cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/SpecimenImportBase.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.specimen;
  11
  12
  13 import java.util.ArrayList;
  14 import java.util.HashMap;
  15 import java.util.HashSet;
  16 import java.util.List;
  17 import java.util.Map;
  18 import java.util.Set;
  19 import java.util.UUID;
  20
  21 import org.apache.log4j.Logger;
  22
  23 import eu.etaxonomy.cdm.api.application.ICdmRepository;
  24 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
  25 import eu.etaxonomy.cdm.api.service.config.FindOccurrencesConfigurator;
  26 import eu.etaxonomy.cdm.api.service.pager.Pager;
  27 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  28 import eu.etaxonomy.cdm.io.common.CdmImportBase;
  29 import eu.etaxonomy.cdm.io.common.IImportConfigurator;
  30 import eu.etaxonomy.cdm.io.specimen.abcd206.in.Abcd206ImportState;
  31 import eu.etaxonomy.cdm.io.specimen.abcd206.in.Identification;
  32 import eu.etaxonomy.cdm.io.specimen.abcd206.in.SpecimenImportReport;
  33 import eu.etaxonomy.cdm.model.agent.AgentBase;
  34 import eu.etaxonomy.cdm.model.agent.Institution;
  35 import eu.etaxonomy.cdm.model.agent.Person;
  36 import eu.etaxonomy.cdm.model.agent.Team;
  37 import eu.etaxonomy.cdm.model.common.CdmBase;
  38 import eu.etaxonomy.cdm.model.common.ISourceable;
  39 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  40 import eu.etaxonomy.cdm.model.common.LanguageString;
  41 import eu.etaxonomy.cdm.model.common.OriginalSourceBase;
  42 import eu.etaxonomy.cdm.model.common.OriginalSourceType;
  43 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  44 import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
  45 import eu.etaxonomy.cdm.model.description.Feature;
  46 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  47 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  48 import eu.etaxonomy.cdm.model.name.BacterialName;
  49 import eu.etaxonomy.cdm.model.name.BotanicalName;
  50 import eu.etaxonomy.cdm.model.name.CultivarPlantName;
  51 import eu.etaxonomy.cdm.model.name.INonViralName;
  52 import eu.etaxonomy.cdm.model.name.ITaxonNameBase;
  53 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  54 import eu.etaxonomy.cdm.model.name.NonViralName;
  55 import eu.etaxonomy.cdm.model.name.Rank;
  56 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
  57 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
  58 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
  59 import eu.etaxonomy.cdm.model.name.TaxonNameFactory;
  60 import eu.etaxonomy.cdm.model.occurrence.Collection;
  61 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  62 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
  63 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
  64 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationType;
  65 import eu.etaxonomy.cdm.model.reference.Reference;
  66 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  67 import eu.etaxonomy.cdm.model.taxon.Classification;
  68 import eu.etaxonomy.cdm.model.taxon.Synonym;
  69 import eu.etaxonomy.cdm.model.taxon.Taxon;
  70 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  71 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  72 import eu.etaxonomy.cdm.persistence.query.MatchMode;
  73 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
  74 import eu.etaxonomy.cdm.strategy.parser.ParserProblem;
  75 import eu.etaxonomy.cdm.strategy.parser.TimePeriodParser;
  76
  77
  78 /**
  79  * @author p.kelbert
  80  * @created 20.10.2008
  81  */
  82 public abstract class SpecimenImportBase<CONFIG extends IImportConfigurator, STATE extends SpecimenImportStateBase>
  83         extends CdmImportBase<CONFIG, STATE> {
  84
  85     private static final long serialVersionUID = 4423065367998125678L;
  86     private static final Logger logger = Logger.getLogger(SpecimenImportBase.class);
  87
  88         protected static final UUID SPECIMEN_SCAN_TERM = UUID.fromString("acda15be-c0e2-4ea8-8783-b9b0c4ad7f03");
  89
  90         private static final String COLON = ":";
  91
  92
  93         @Override
  94     protected abstract void doInvoke(STATE state);
  95
  96         /**
  97          * Handle a single unit
  98          * @param state
  99          * @param item
 100          */
 101         protected abstract void handleSingleUnit(STATE state, Object item, boolean handleAssociatedUnits) ;
 102
 103
 104
 105         protected TaxonNameBase<?, ?> getOrCreateTaxonName(String scientificName, Rank rank, boolean preferredFlag, STATE state, int unitIndexInAbcdFile){
 106             TaxonNameBase<?, ?> taxonName = null;
 107         SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
 108
 109         //check atomised name data for rank
 110         //new name will be created
 111         ITaxonNameBase atomisedTaxonName = null;
 112         if (rank==null && unitIndexInAbcdFile>=0 && ((state.getDataHolder().getAtomisedIdentificationList() != null && !state.getDataHolder().getAtomisedIdentificationList().isEmpty())|| state.getDataHolder().getAtomisedIdentificationList().size() > 0)) {
 113             atomisedTaxonName = setTaxonNameByType(state.getDataHolder().getAtomisedIdentificationList().get(unitIndexInAbcdFile), scientificName, state);
 114             if(atomisedTaxonName!=null){
 115                 rank = atomisedTaxonName.getRank();
 116             }
 117         }
 118         if(config.isReuseExistingTaxaWhenPossible()){
 119             ITaxonNameBase parsedName = atomisedTaxonName;
 120             if(parsedName==null){
 121
 122                 parsedName = parseScientificName(scientificName, state, state.getReport(), rank);
 123
 124             }
 125             atomisedTaxonName = parsedName;
 126             if(config.isIgnoreAuthorship() && parsedName!=null){// && preferredFlag){
 127                 // do not ignore authorship for non-preferred names because they need
 128                 // to be created for the determination history
 129                 String nameCache = TaxonNameBase.castAndDeproxy(parsedName).getNameCache();
 130                 List<NonViralName> names = getNameService().findNamesByNameCache(nameCache, MatchMode.EXACT, null);
 131                 if (!names.isEmpty()){
 132                      taxonName = getBestMatchingName(scientificName, new ArrayList<TaxonNameBase>(names), state);
 133                 }
 134                 if (taxonName == null && !names.isEmpty()){
 135                     taxonName = names.get(0);
 136                 }
 137
 138             } else {
 139                 //search for existing names
 140                 List<TaxonNameBase> names = getNameService().listByTitle(TaxonNameBase.class, scientificName, MatchMode.EXACT, null, null, null, null, null);
 141                 taxonName = getBestMatchingName(scientificName, names, state);
 142                 //still nothing found -> try with the atomised name full title cache
 143                 if(taxonName==null && atomisedTaxonName!=null){
 144                     names = getNameService().listByTitle(TaxonNameBase.class, atomisedTaxonName.getFullTitleCache(), MatchMode.EXACT, null, null, null, null, null);
 145                     taxonName = getBestMatchingName(atomisedTaxonName.getTitleCache(), names, state);
 146                     //still nothing found -> try with the atomised name title cache
 147                     if(taxonName==null){
 148                         names = getNameService().listByTitle(TaxonNameBase.class, atomisedTaxonName.getTitleCache(), MatchMode.EXACT, null, null, null, null, null);
 149                         taxonName = getBestMatchingName(atomisedTaxonName.getTitleCache(), names, state);
 150                     }
 151                 }
 152
 153             }
 154
 155         }
 156
 157         if(taxonName==null && atomisedTaxonName!=null){
 158             taxonName = (TaxonNameBase<?, ?>) atomisedTaxonName;
 159             state.getReport().addName(taxonName);
 160             logger.info("Created new taxon name "+taxonName);
 161             if(taxonName.hasProblem()){
 162                 state.getReport().addInfoMessage(String.format("Created %s with parsing problems", taxonName));
 163             }
 164             if(!atomisedTaxonName.getTitleCache().equals(scientificName)){
 165                 state.getReport().addInfoMessage(String.format("Taxon %s was parsed as %s", scientificName, atomisedTaxonName.getTitleCache()));
 166             }
 167         }
 168         else if(taxonName==null){
 169             //create new taxon name
 170
 171             if (state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNAFP)){
 172                 taxonName = TaxonNameFactory.NewBotanicalInstance(rank);
 173             }else if (state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICZN)){
 174                 taxonName = TaxonNameFactory.NewZoologicalInstance(rank);
 175             }else{
 176                 taxonName = TaxonNameFactory.NewNonViralInstance(rank);
 177             }
 178             taxonName.setFullTitleCache(scientificName,true);
 179             taxonName.setTitleCache(scientificName, true);
 180             state.getReport().addName(taxonName);
 181             logger.info("Created new taxon name "+taxonName);
 182         }
 183         save(taxonName, state);
 184         return taxonName;
 185     }
 186
 187          protected TaxonNameBase<?, ?> getBestMatchingName(String scientificName, java.util.Collection<TaxonNameBase> names, STATE state){
 188                 Set<TaxonNameBase> namesWithAcceptedTaxa = new HashSet<TaxonNameBase>();
 189                 List<TaxonNameBase> namesWithAcceptedTaxaInClassification = new ArrayList<TaxonNameBase>();
 190                 for (TaxonNameBase name : names) {
 191                     if(!name.getTaxa().isEmpty()){
 192                         Set<Taxon> taxa = name.getTaxa();
 193                         for (Taxon taxon:taxa){
 194                             if (!taxon.getTaxonNodes().isEmpty()){
 195                                 //use only taxa included in a classification
 196                                 for (TaxonNode node:taxon.getTaxonNodes()){
 197                                     if (state.getClassification() != null && node.getClassification().equals(state.getClassification())){
 198                                         namesWithAcceptedTaxaInClassification.add(name);
 199                                     }else {
 200                                         namesWithAcceptedTaxa.add(name);
 201                                     }
 202                                 }
 203
 204                             }
 205                         }
 206
 207                     }
 208                 }
 209                 String message = String.format("More than one taxon name was found for %s, maybe in other classifications!", scientificName);
 210                 //check for names with accepted taxa in classification
 211                 if(namesWithAcceptedTaxaInClassification.size()>0){
 212                 if(namesWithAcceptedTaxaInClassification.size()>1){
 213
 214                     state.getReport().addInfoMessage(message);
 215                     logger.warn(message);
 216                     return null;
 217                 }
 218                 return namesWithAcceptedTaxaInClassification.iterator().next();
 219             }
 220               //check for any names with accepted taxa
 221                 if(namesWithAcceptedTaxa.size()>0){
 222                     if(namesWithAcceptedTaxa.size()>1){
 223
 224                         state.getReport().addInfoMessage(message);
 225                         logger.warn(message);
 226                         return null;
 227                     }
 228                     return namesWithAcceptedTaxa.iterator().next();
 229                 }
 230                 //no names with accepted taxa found -> check accepted taxa of synonyms
 231                 List<Taxon> taxaFromSynonyms = new ArrayList<>();
 232                 for (TaxonNameBase name : names) {
 233                     Set<TaxonBase> taxonBases = name.getTaxonBases();
 234                     for (TaxonBase taxonBase : taxonBases) {
 235                         if(taxonBase.isInstanceOf(Synonym.class)){
 236                             Synonym synonym = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
 237                             taxaFromSynonyms.add(synonym.getAcceptedTaxon());
 238                         }
 239                     }
 240                 }
 241                 if(taxaFromSynonyms.size()>0){
 242                     if(taxaFromSynonyms.size()>1){
 243                         state.getReport().addInfoMessage(message);
 244                         logger.warn(message);
 245                         return null;
 246                     }
 247                     return taxaFromSynonyms.iterator().next().getName();
 248                 }
 249                 //no accepted and no synonyms -> return one of the names and create a new taxon
 250                 if (names.isEmpty()){
 251                     return null;
 252                 }else{
 253                     return names.iterator().next();
 254                 }
 255             }
 256          /**
 257              * Parse automatically the scientific name
 258              * @param scientificName the scientific name to parse
 259              * @param state the current import state
 260              * @param report the import report
 261              * @return a parsed name
 262              */
 263
 264             protected ITaxonNameBase parseScientificName(String scientificName, STATE state, SpecimenImportReport report, Rank rank) {
 265
 266                 NonViralNameParserImpl nvnpi = NonViralNameParserImpl.NewInstance();
 267                 ITaxonNameBase taxonName = null;
 268                 boolean problem = false;
 269
 270                 if (logger.isDebugEnabled()){
 271                     logger.debug("parseScientificName " + state.getDataHolder().getNomenclatureCode().toString());
 272                 }
 273
 274                 if (state.getDataHolder().getNomenclatureCode().toString().equals("Zoological") || state.getDataHolder().getNomenclatureCode().toString().contains("ICZN")) {
 275                     taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICZN, rank);
 276                     if (taxonName.hasProblem()) {
 277                         problem = true;
 278                     }
 279                 }
 280                 else if (state.getDataHolder().getNomenclatureCode().toString().equals("Botanical") || state.getDataHolder().getNomenclatureCode().toString().contains("ICBN")) {
 281                     taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNAFP, rank);
 282                     if (taxonName.hasProblem()) {
 283                         problem = true;
 284                     }
 285                 }
 286                 else if (state.getDataHolder().getNomenclatureCode().toString().equals("Bacterial") || state.getDataHolder().getNomenclatureCode().toString().contains("ICBN")) {
 287                     taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNB, rank);
 288                     if (taxonName.hasProblem()) {
 289                         problem = true;
 290                     }
 291                 }
 292                 else if (state.getDataHolder().getNomenclatureCode().toString().equals("Cultivar") || state.getDataHolder().getNomenclatureCode().toString().contains("ICNCP")) {
 293                     taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNCP, rank);
 294                     if (taxonName.hasProblem()) {
 295                         problem = true;
 296                     }
 297                 }
 298                 if (problem) {
 299                     String message = String.format("Parsing problems for %s", scientificName);
 300                     if(taxonName!=null){
 301                         for (ParserProblem parserProblem : taxonName.getParsingProblems()) {
 302                             message += "\n\t- "+parserProblem;
 303                         }
 304                     }
 305                     report.addInfoMessage(message);
 306                     logger.info(message);
 307                 }
 308                 return taxonName;
 309
 310             }
 311
 312             /**
 313              * Create the name without automatic parsing, either because it failed, or because the user deactivated it.
 314              * The name is built upon the ABCD fields
 315              * @param atomisedMap : the ABCD atomised fields
 316              * @param fullName : the full scientific name
 317              * @param state
 318              * @return the corresponding Botanical or Zoological or... name
 319              */
 320             protected TaxonNameBase<?,?> setTaxonNameByType(
 321                     HashMap<String, String> atomisedMap, String fullName, STATE state) {
 322                 boolean problem = false;
 323                 if (logger.isDebugEnabled()){
 324                     logger.debug("settaxonnamebytype " + state.getDataHolder().getNomenclatureCode().toString());
 325                 }
 326
 327                 if (state.getDataHolder().getNomenclatureCode().equals("Zoological") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICZN.getUuid())) {
 328                     TaxonNameBase<?,?> taxonName = TaxonNameFactory.NewZoologicalInstance(null);
 329                     taxonName.setFullTitleCache(fullName, true);
 330                     taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
 331                     taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
 332                     taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap,"SpeciesEpithet")));
 333                     taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap,"SubspeciesEpithet")));
 334
 335                     if (taxonName.getGenusOrUninomial() != null){
 336                         taxonName.setRank(Rank.GENUS());
 337                     }
 338
 339                     if (taxonName.getInfraGenericEpithet() != null){
 340                         taxonName.setRank(Rank.SUBGENUS());
 341                     }
 342
 343                     if (taxonName.getSpecificEpithet() != null){
 344                         taxonName.setRank(Rank.SPECIES());
 345                     }
 346
 347                     if (taxonName.getInfraSpecificEpithet() != null){
 348                         taxonName.setRank(Rank.SUBSPECIES());
 349                     }
 350
 351                     Team team = null;
 352                     if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 353                         team = Team.NewInstance();
 354                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
 355                     }
 356                     else {
 357                         if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 358                             team = Team.NewInstance();
 359                             team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
 360                         }
 361                     }
 362                     if (team != null) {
 363                         taxonName.setBasionymAuthorship(team);
 364                     }
 365                     else {
 366                         if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 367                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
 368                         }
 369                         else if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 370                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamAndYear"));
 371                         }
 372                     }
 373                     if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
 374                         team = Team.NewInstance();
 375                         team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
 376                         taxonName.setCombinationAuthorship(team);
 377                     }
 378                     if (taxonName.hasProblem()) {
 379                         logger.info("pb ICZN");
 380                         problem = true;
 381                     }
 382                     else {
 383                         return taxonName;
 384                     }
 385                 }
 386                 else if (state.getDataHolder().getNomenclatureCode().equals("Botanical") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNAFP.getUuid())) {
 387                     BotanicalName taxonName = (BotanicalName) parseScientificName(fullName, state, state.getReport(), null);
 388                     if (taxonName != null){
 389                         return taxonName;
 390                     }
 391                     else{
 392                         taxonName = TaxonNameFactory.NewBotanicalInstance(null);
 393                     }
 394                     taxonName.setFullTitleCache(fullName, true);
 395                     taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
 396                     taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "FirstEpithet")));
 397                     taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "InfraSpeEpithet")));
 398                     try {
 399                         taxonName.setRank(Rank.getRankByName(getFromMap(atomisedMap, "Rank")));
 400                     } catch (Exception e) {
 401                         if (taxonName.getInfraSpecificEpithet() != null){
 402                             taxonName.setRank(Rank.SUBSPECIES());
 403                         }
 404                         else if (taxonName.getSpecificEpithet() != null){
 405                             taxonName.setRank(Rank.SPECIES());
 406                         }
 407                         else if (taxonName.getInfraGenericEpithet() != null){
 408                             taxonName.setRank(Rank.SUBGENUS());
 409                         }
 410                         else if (taxonName.getGenusOrUninomial() != null){
 411                             taxonName.setRank(Rank.GENUS());
 412                         }
 413                     }
 414                     Team team = null;
 415                     if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 416                         team = Team.NewInstance();
 417                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
 418                         taxonName.setBasionymAuthorship(team);
 419                     }
 420                     if (getFromMap(atomisedMap, "AuthorTeam") != null) {
 421                         team = Team.NewInstance();
 422                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeam"), true);
 423                         taxonName.setCombinationAuthorship(team);
 424                     }
 425                     if (team == null) {
 426                         if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 427                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
 428                         }
 429                         else if (getFromMap(atomisedMap, "AuthorTeam") != null) {
 430                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeam"));
 431                         }
 432                     }
 433                     if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
 434                         team = Team.NewInstance();
 435                         team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
 436                         taxonName.setCombinationAuthorship(team);
 437                     }
 438                     if (taxonName.hasProblem()) {
 439                         logger.info("pb ICBN");
 440                         problem = true;
 441                     }
 442                     else {
 443                         return taxonName;
 444                     }
 445                 }
 446                 else if (state.getDataHolder().getNomenclatureCode().equals("Bacterial") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNB.getUuid())) {
 447                     NonViralName<BacterialName> taxonName = TaxonNameFactory.NewBacterialInstance(null);
 448                     taxonName.setFullTitleCache(fullName, true);
 449                     taxonName.setGenusOrUninomial(getFromMap(atomisedMap, "Genus"));
 450                     taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
 451                     taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "Species")));
 452                     taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "SubspeciesEpithet")));
 453
 454                     if (taxonName.getGenusOrUninomial() != null){
 455                         taxonName.setRank(Rank.GENUS());
 456                     }
 457                     else if (taxonName.getInfraGenericEpithet() != null){
 458                         taxonName.setRank(Rank.SUBGENUS());
 459                     }
 460                     else if (taxonName.getSpecificEpithet() != null){
 461                         taxonName.setRank(Rank.SPECIES());
 462                     }
 463                     else if (taxonName.getInfraSpecificEpithet() != null){
 464                         taxonName.setRank(Rank.SUBSPECIES());
 465                     }
 466
 467                     if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 468                         Team team = Team.NewInstance();
 469                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
 470                         taxonName.setCombinationAuthorship(team);
 471                     }
 472                     if (getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear") != null) {
 473                         Team team = Team.NewInstance();
 474                         team.setTitleCache(getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear"), true);
 475                         taxonName.setBasionymAuthorship(team);
 476                     }
 477                     if (taxonName.hasProblem()) {
 478                         logger.info("pb ICNB");
 479                         problem = true;
 480                     }
 481                     else {
 482                         return taxonName;
 483                     }
 484                 }
 485                 else if (state.getDataHolder().getNomenclatureCode().equals("Cultivar")) {
 486                     CultivarPlantName taxonName = TaxonNameFactory.NewCultivarInstance(null);
 487
 488                     if (taxonName.hasProblem()) {
 489                         logger.info("pb ICNCP");
 490                         problem = true;
 491                     }
 492                     else {
 493                         return taxonName;
 494                     }
 495                     return taxonName;
 496                 }
 497
 498                 if (problem) {
 499                     logger.info("Problem im setTaxonNameByType ");
 500                     TaxonNameBase<?,?> taxonName = TaxonNameFactory.NewNonViralInstance(null);
 501                     taxonName.setFullTitleCache(fullName, true);
 502                     return taxonName;
 503                 }
 504                 TaxonNameBase<?,?> tn = TaxonNameFactory.NewNonViralInstance(null);
 505                 return tn;
 506             }
 507
 508             /**
 509              * Get a formated string from a hashmap
 510              * @param atomisedMap
 511              * @param key
 512              * @return
 513              */
 514             private String getFromMap(HashMap<String, String> atomisedMap, String key) {
 515                 String value = null;
 516                 if (atomisedMap.containsKey(key)) {
 517                     value = atomisedMap.get(key);
 518                 }
 519
 520                 try {
 521                     if (value != null && key.matches(".*Year.*")) {
 522                         value = value.trim();
 523                         if (value.matches("[a-z A-Z ]*[0-9]{4}$")) {
 524                             String tmp = value.split("[0-9]{4}$")[0];
 525                             int year = Integer.parseInt(value.split(tmp)[1]);
 526                             if (year >= 1752) {
 527                                 value = tmp;
 528                             }
 529                             else {
 530                                 value = null;
 531                             }
 532                         }
 533                         else {
 534                             value = null;
 535                         }
 536                     }
 537                 }
 538                 catch (Exception e) {
 539                     value = null;
 540                 }
 541                 return value;
 542             }
 543
 544             /**
 545              * Very fast and dirty implementation to allow handling of transient objects as described in
 546              * https://dev.e-taxonomy.eu/trac/ticket/3726
 547              *
 548              * Not yet complete.
 549              *
 550              * @param cdmBase
 551              * @param state
 552              */
 553             protected UUID save(CdmBase cdmBase, SpecimenImportStateBase state) {
 554                 ICdmRepository cdmRepository = state.getConfig().getCdmAppController();
 555                 if (cdmRepository == null){
 556                     cdmRepository = this;
 557                 }
 558
 559                 if (cdmBase.isInstanceOf(LanguageString.class)){
 560                     return cdmRepository.getTermService().saveLanguageData(CdmBase.deproxy(cdmBase, LanguageString.class));
 561                 }else if (cdmBase.isInstanceOf(SpecimenOrObservationBase.class)){
 562                     return cdmRepository.getOccurrenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, SpecimenOrObservationBase.class));
 563                 }else if (cdmBase.isInstanceOf(Reference.class)){
 564                     return cdmRepository.getReferenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, Reference.class));
 565                 }else if (cdmBase.isInstanceOf(Classification.class)){
 566                     return cdmRepository.getClassificationService().saveOrUpdate(CdmBase.deproxy(cdmBase, Classification.class));
 567                 }else if (cdmBase.isInstanceOf(AgentBase.class)){
 568                     return cdmRepository.getAgentService().saveOrUpdate(CdmBase.deproxy(cdmBase, AgentBase.class));
 569                 }else if (cdmBase.isInstanceOf(Collection.class)){
 570                     return cdmRepository.getCollectionService().saveOrUpdate(CdmBase.deproxy(cdmBase, Collection.class));
 571                 }else if (cdmBase.isInstanceOf(DescriptionBase.class)){
 572                     return cdmRepository.getDescriptionService().saveOrUpdate(CdmBase.deproxy(cdmBase, DescriptionBase.class));
 573                 }else if (cdmBase.isInstanceOf(TaxonBase.class)){
 574                     return cdmRepository.getTaxonService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonBase.class));
 575                 }else if (cdmBase.isInstanceOf(TaxonNameBase.class)){
 576                     return cdmRepository.getNameService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonNameBase.class));
 577                 }else if (cdmBase.isInstanceOf(TaxonNode.class)){
 578                 return cdmRepository.getTaxonNodeService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonNode.class));
 579             }else{
 580                     throw new IllegalArgumentException("Class not supported in save method: " + CdmBase.deproxy(cdmBase, CdmBase.class).getClass().getSimpleName());
 581                 }
 582
 583             }
 584
 585
 586             protected SpecimenOrObservationBase findExistingSpecimen(String unitId, SpecimenImportStateBase state){
 587                 ICdmRepository cdmAppController = state.getConfig().getCdmAppController();
 588                 if(cdmAppController==null){
 589                     cdmAppController = this;
 590                 }
 591                 FindOccurrencesConfigurator config = new FindOccurrencesConfigurator();
 592                 config.setSignificantIdentifier(unitId);
 593                 Pager<SpecimenOrObservationBase> existingSpecimens = cdmAppController.getOccurrenceService().findByTitle(config);
 594                 if(!existingSpecimens.getRecords().isEmpty()){
 595                     if(existingSpecimens.getRecords().size()==1){
 596                         return existingSpecimens.getRecords().iterator().next();
 597                     }
 598                 }
 599                 return null;
 600             }
 601
 602             protected abstract void importAssociatedUnits(STATE state, Object item, DerivedUnitFacade derivedUnitFacade);
 603
 604             /**
 605              * getFacade : get the DerivedUnitFacade based on the recordBasis
 606              * @param state
 607              *
 608              * @return DerivedUnitFacade
 609              */
 610             protected DerivedUnitFacade getFacade(STATE state) {
 611                 if (logger.isDebugEnabled()){
 612                     logger.info("getFacade()");
 613                 }
 614                 SpecimenOrObservationType type = null;
 615
 616                 // create specimen
 617                 if (NB((state.getDataHolder().getRecordBasis())) != null) {
 618                     if (state.getDataHolder().getRecordBasis().toLowerCase().startsWith("s") || state.getDataHolder().getRecordBasis().toLowerCase().indexOf("specimen")>-1) {// specimen
 619                         type = SpecimenOrObservationType.PreservedSpecimen;
 620                     }
 621                     if (state.getDataHolder().getRecordBasis().toLowerCase().startsWith("o") ||state.getDataHolder().getRecordBasis().toLowerCase().indexOf("observation")>-1 ) {
 622                         type = SpecimenOrObservationType.Observation;
 623                     }
 624                     if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("fossil")>-1){
 625                         type = SpecimenOrObservationType.Fossil;
 626                     }
 627                     if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("living")>-1) {
 628                         type = SpecimenOrObservationType.LivingSpecimen;
 629                     }
 630                     if (type == null) {
 631                         logger.info("The basis of record does not seem to be known: " + state.getDataHolder().getRecordBasis());
 632                         type = SpecimenOrObservationType.DerivedUnit;
 633                     }
 634                     // TODO fossils?
 635                 } else {
 636                     logger.info("The basis of record is null");
 637                     type = SpecimenOrObservationType.DerivedUnit;
 638                 }
 639                 DerivedUnitFacade derivedUnitFacade = DerivedUnitFacade.NewInstance(type);
 640                 return derivedUnitFacade;
 641             }
 642
 643             /**
 644              * Look if the Institution does already exist
 645              * @param institutionCode: a string with the institutioncode
 646              * @param config : the configurator
 647              * @return the Institution (existing or new)
 648              */
 649             protected Institution getInstitution(String institutionCode, STATE state) {
 650                 SpecimenImportConfiguratorBase config = state.getConfig();
 651                 Institution institution=null;
 652                 institution = (Institution)state.institutions.get(institutionCode);
 653                 if (institution != null){
 654                     return institution;
 655                 }
 656                 List<Institution> institutions;
 657                 try {
 658                     institutions = getAgentService().searchInstitutionByCode(institutionCode);
 659
 660                 } catch (Exception e) {
 661                     institutions = new ArrayList<Institution>();
 662                     logger.warn(e);
 663                 }
 664                 if (institutions.size() > 0 && config.isReuseExistingMetaData()) {
 665                     for (Institution institut:institutions){
 666                         try{
 667                             if (institut.getCode().equalsIgnoreCase(institutionCode)) {
 668                                 institution=institut;
 669                                 break;
 670                             }
 671                         }catch(Exception e){logger.warn("no institution code in the db");}
 672                     }
 673                 }
 674                 if (logger.isDebugEnabled()){
 675                     if(institution !=null) {
 676                         logger.info("getinstitution " + institution.toString());
 677                     }
 678                 }
 679                 if (institution == null){
 680                     // create institution
 681                     institution = Institution.NewInstance();
 682                     institution.setCode(institutionCode);
 683                     institution.setTitleCache(institutionCode, true);
 684                     UUID uuid = save(institution, state);
 685                 }
 686
 687
 688                 state.institutions.put(institutionCode, institution);
 689                 return institution;
 690             }
 691
 692             /**
 693              * Look if the Collection does already exist
 694              * @param collectionCode
 695              * @param collectionCode: a string
 696              * @param config : the configurator
 697              * @return the Collection (existing or new)
 698              */
 699             protected Collection getCollection(Institution institution, String collectionCode, STATE state) {
 700                 SpecimenImportConfiguratorBase config = state.getConfig();
 701                 Collection collection = null;
 702                 List<Collection> collections;
 703                 collection = (Collection) state.collections.get(collectionCode);
 704                 if (collection != null){
 705                     return collection;
 706                 }
 707                 try {
 708                     collections = getCollectionService().searchByCode(collectionCode);
 709                 } catch (Exception e) {
 710                     collections = new ArrayList<Collection>();
 711                 }
 712                 if (collections.size() > 0 && config.isReuseExistingMetaData()) {
 713                     for (Collection coll:collections){
 714                         if (coll.getCode() != null && coll.getInstitute() != null
 715                                 && coll.getCode().equalsIgnoreCase(collectionCode) && coll.getInstitute().equals(institution)) {
 716                             collection = coll;
 717                             break;
 718                         }
 719                     }
 720                 }
 721
 722                 if(collection == null){
 723                     collection =Collection.NewInstance();
 724                     collection.setCode(collectionCode);
 725                     collection.setInstitute(institution);
 726                     collection.setTitleCache(collectionCode);
 727                     UUID uuid = save(collection, state);
 728                 }
 729
 730
 731
 732                 state.collections.put(collectionCode, collection);
 733
 734                 return collection;
 735             }
 736
 737             /**
 738              * @param reference
 739              * @param citationDetail
 740              * @return
 741              */
 742             //FIXME this method is highly critical, because
 743             //  * it will have serious performance and memory problems with large databases
 744             //        (databases may easily have >1 Mio source records)
 745             //  * it does not make sense to search for existing sources and then clone them
 746             //    we need to search for existing references instead and use them (if exist)
 747             //    for our new source.
 748             protected IdentifiableSource getIdentifiableSource(Reference reference, String citationDetail) {
 749
 750               /*  List<IdentifiableSource> issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
 751
 752
 753                 if (reference != null){
 754                     try {
 755                         for (OriginalSourceBase<?> osb: issTmp){
 756                             if (osb.getCitation() != null && osb.getCitation().getTitleCache().equalsIgnoreCase(reference.getTitleCache())){
 757                                 String osbDetail = osb.getCitationMicroReference();
 758                                 if ((StringUtils.isBlank(osbDetail) && StringUtils.isBlank(citationDetail))
 759                                         || (osbDetail != null && osbDetail.equalsIgnoreCase(citationDetail)) ) {
 760 //                                  System.out.println("REFERENCE FOUND RETURN EXISTING SOURCE");
 761                                     return (IdentifiableSource) osb.clone();
 762                                 }
 763                             }
 764                         }
 765                     } catch (CloneNotSupportedException e) {
 766                         throw new RuntimeException(e);
 767                     } catch (Exception e1){
 768                         e1.printStackTrace();
 769                     }
 770                 }
 771         */
 772                 IdentifiableSource sour = IdentifiableSource.NewInstance(OriginalSourceType.Import,null,null, reference,citationDetail);
 773                 return sour;
 774             }
 775
 776             /**
 777              * Add the hierarchy for a Taxon(add higher taxa)
 778              * @param classification
 779              * @param taxon: a taxon to add as a node
 780              * @param state: the ABCD import state
 781              */
 782             protected void addParentTaxon(Taxon taxon, STATE state, boolean preferredFlag, Classification classification){
 783                 INonViralName  nvname = taxon.getName();
 784                 Rank rank = nvname.getRank();
 785                 Taxon genus =null;
 786                 Taxon subgenus =null;
 787                 Taxon species = null;
 788                 Taxon subspecies = null;
 789                 Taxon parent = null;
 790                 if(rank!=null){
 791                     if (rank.isLower(Rank.GENUS() )){
 792                         String genusOrUninomial = nvname.getGenusOrUninomial();
 793                         TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(genusOrUninomial, Rank.GENUS(), preferredFlag, state, -1);
 794                         genus = getOrCreateTaxonForName(taxonName, state);
 795                         if (genus == null){
 796                             logger.debug("The genus should not be null " + taxonName);
 797                         }
 798                         if (preferredFlag) {
 799                             parent = linkParentChildNode(null, genus, classification, state);
 800                         }
 801
 802                     }
 803                     if (rank.isLower(Rank.SUBGENUS())){
 804                         String prefix = nvname.getGenusOrUninomial();
 805                         String name = nvname.getInfraGenericEpithet();
 806                         if (name != null){
 807                             TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(prefix+" "+name, Rank.SUBGENUS(), preferredFlag, state, -1);
 808                             subgenus = getOrCreateTaxonForName(taxonName, state);
 809                             if (preferredFlag) {
 810                                 parent = linkParentChildNode(genus, subgenus, classification, state);
 811                             }            }
 812                     }
 813                     if (rank.isLower(Rank.SPECIES())){
 814                         if (subgenus!=null){
 815                             String prefix = nvname.getGenusOrUninomial();
 816                             String name = nvname.getInfraGenericEpithet();
 817                             String spe = nvname.getSpecificEpithet();
 818                             if (spe != null){
 819                                 TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(prefix+" "+name+" "+spe, Rank.SPECIES(), preferredFlag, state, -1);
 820                                 species = getOrCreateTaxonForName(taxonName, state);
 821                                 if (preferredFlag) {
 822                                     parent = linkParentChildNode(subgenus, species, classification, state);
 823                                 }
 824                             }
 825                         }
 826                         else{
 827                             String prefix = nvname.getGenusOrUninomial();
 828                             String name = nvname.getSpecificEpithet();
 829                             if (name != null){
 830                                 TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(prefix+" "+name, Rank.SPECIES(), preferredFlag, state, -1);
 831                                 species = getOrCreateTaxonForName(taxonName, state);
 832                                 if (preferredFlag) {
 833                                     parent = linkParentChildNode(genus, species, classification, state);
 834                                 }
 835                             }
 836                         }
 837                     }
 838                     if (rank.isLower(Rank.INFRASPECIES())){
 839                         TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(nvname.getFullTitleCache(), Rank.SUBSPECIES(), preferredFlag, state, -1);
 840                         subspecies = getOrCreateTaxonForName(taxonName, state);
 841                         if (preferredFlag) {
 842                             parent = linkParentChildNode(species, subspecies, classification, state);
 843                         }
 844                     }
 845                 }
 846                 if (preferredFlag && parent!=taxon ) {
 847                     linkParentChildNode(parent, taxon, classification, state);
 848                 }
 849             }
 850
 851             /**
 852              * Link a parent to a child and save it in the current classification
 853              * @param parent: the higher Taxon
 854              * @param child : the lower (or current) Taxon
 855              * return the Taxon from the new created Node
 856              * @param classification
 857              * @param state
 858              */
 859             protected Taxon linkParentChildNode(Taxon parent, Taxon child, Classification classification, STATE state) {
 860                 TaxonNode node =null;
 861                 if (parent != null) {
 862                     parent = (Taxon) getTaxonService().find(parent.getUuid());
 863                     child = (Taxon) getTaxonService().find(child.getUuid());
 864                     //here we do not have to check if the taxon nodes already exists
 865                     //this is done by classification.addParentChild()
 866                     //do not add child node if it already exists
 867                     if(hasTaxonNodeInClassification(child, classification)){
 868                         return child;
 869                     }
 870                     else{
 871                         node = classification.addParentChild(parent, child, state.getRef(), "");
 872                         save(node, state);
 873                     }
 874                 }
 875                 else {
 876                     if (child == null){
 877                         logger.debug("The child should not be null!");
 878                     }
 879                     child = (Taxon) getTaxonService().find(child.getUuid());
 880                     //do not add child node if it already exists
 881                     if(hasTaxonNodeInClassification(child, classification)){
 882                         return child;
 883                     }
 884                     else{
 885                         node = classification.addChildTaxon(child, state.getRef(), null);
 886                         save(node, state);
 887                     }
 888                 }
 889                 if(node!=null){
 890                     state.getReport().addTaxonNode(node);
 891                     return node.getTaxon();
 892                 }
 893                 String message = "Could not create taxon node for " +child;
 894                 state.getReport().addInfoMessage(message);
 895                 logger.warn(message);
 896                 return null;
 897             }
 898
 899             protected Taxon getOrCreateTaxonForName(TaxonNameBase<?, ?> taxonNameBase, STATE state){
 900                 if (taxonNameBase != null){
 901                 Set<Taxon> acceptedTaxa = taxonNameBase.getTaxa();
 902                 if(acceptedTaxa.size()>0){
 903                     Taxon firstAcceptedTaxon = acceptedTaxa.iterator().next();
 904                     if(acceptedTaxa.size()>1){
 905                         String message = "More than one accepted taxon was found for taxon name: "
 906                                 + taxonNameBase.getTitleCache() + "!\n" + firstAcceptedTaxon + "was chosen for "+state.getDerivedUnitBase();
 907                         state.getReport().addInfoMessage(message);
 908                         logger.warn(message);
 909                     }
 910                     else{
 911                         return firstAcceptedTaxon;
 912                     }
 913                 }
 914                 else{
 915                     Set<TaxonBase> taxonAndSynonyms = taxonNameBase.getTaxonBases();
 916                     for (TaxonBase taxonBase : taxonAndSynonyms) {
 917                         if(taxonBase.isInstanceOf(Synonym.class)){
 918                             Synonym synonym = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
 919                             Taxon acceptedTaxonOfSynonym = synonym.getAcceptedTaxon();
 920                             if(acceptedTaxonOfSynonym == null){
 921                                 String message = "No accepted taxon could be found for taxon name: "
 922                                         + taxonNameBase.getTitleCache()
 923                                         + "!";
 924                                 state.getReport().addInfoMessage(message);
 925                                 logger.warn(message);
 926                             }
 927                             else{
 928                                 return acceptedTaxonOfSynonym;
 929                             }
 930                         }
 931                     }
 932                 }
 933                 Taxon taxon = Taxon.NewInstance(taxonNameBase, state.getRef());
 934                 save(taxon, state);
 935                 state.getReport().addTaxon(taxon);
 936                 logger.info("Created new taxon "+ taxon);
 937                 return taxon;
 938                 }
 939                 return null;
 940
 941             }
 942
 943             private boolean hasTaxonNodeInClassification(Taxon taxon, Classification classification){
 944                 if(taxon.getTaxonNodes()!=null){
 945                     for (TaxonNode node : taxon.getTaxonNodes()){
 946                         if(node.getClassification().equals(classification)){
 947                             return true;
 948                         }
 949                     }
 950                 }
 951                 return false;
 952             }
 953
 954             /**
 955              * HandleIdentifications : get the scientific names present in the ABCD
 956              * document and store link them with the observation/specimen data
 957              * @param state: the current ABCD import state
 958              * @param derivedUnitFacade : the current derivedunitfacade
 959              */
 960             protected void handleIdentifications(STATE state, DerivedUnitFacade derivedUnitFacade) {
 961                SpecimenImportConfiguratorBase config = state.getConfig();
 962
 963
 964                 String scientificName = "";
 965                 boolean preferredFlag = false;
 966
 967                 if (state.getDataHolder().getNomenclatureCode() == ""){
 968                     if (config.getNomenclaturalCode() != null){
 969                         if (config.getNomenclaturalCode() != null){
 970                             state.getDataHolder().setNomenclatureCode(config.getNomenclaturalCode().toString());
 971                         }
 972                     }
 973                 }
 974
 975                 for (int i = 0; i < state.getDataHolder().getIdentificationList().size(); i++) {
 976                     Identification identification = state.getDataHolder().getIdentificationList().get(i);
 977                     scientificName = identification.getScientificName().replaceAll(" et ", " & ");
 978
 979                     String preferred = identification.getPreferred();
 980                     preferredFlag = false;
 981                     if (preferred != null){
 982                     if (preferred.equals("1") || preferred.toLowerCase().indexOf("true") != -1 || state.getDataHolder().getIdentificationList().size()==1) {
 983                         preferredFlag = true;
 984                     }
 985
 986                     }
 987                     if (identification.getCode() != null){
 988                     if (identification.getCode().indexOf(':') != -1) {
 989                         state.getDataHolder().setNomenclatureCode(identification.getCode().split(COLON)[1]);
 990                     }
 991                     else{
 992                         state.getDataHolder().setNomenclatureCode(identification.getCode());
 993                     }
 994                     }
 995                     TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(scientificName, null, preferredFlag, state, i);
 996                     Taxon taxon = getOrCreateTaxonForName(taxonName, state);
 997                     addTaxonNode(taxon, state,preferredFlag);
 998                     linkDeterminationEvent(state, taxon, preferredFlag, derivedUnitFacade, identification.getIdentifier(), identification.getDate());
 999                 }
1000             }
1001
1002             /**
1003              * @param taxon : a taxon to add as a node
1004              * @param state : the ABCD import state
1005              */
1006             protected void addTaxonNode(Taxon taxon, STATE state, boolean preferredFlag) {
1007                 SpecimenImportConfiguratorBase config = state.getConfig();
1008                 logger.info("link taxon to a taxonNode "+taxon.getTitleCache());
1009                 //only add nodes if not already existing in current classification or default classification
1010
1011                 //check if node exists in current classification
1012                 //NOTE: we cannot use hasTaxonNodeInClassification() here because we are first creating it here
1013                 if (!existsInClassification(taxon,state.getClassification(), state)){
1014                     if(config.isMoveNewTaxaToDefaultClassification()){
1015                         //check if node exists in default classification
1016                         if (!existsInClassification(taxon, state.getDefaultClassification(), state)){
1017                             addParentTaxon(taxon, state, preferredFlag, state.getDefaultClassification());
1018                         }
1019                     }
1020                     else {
1021                         //add non-existing taxon to current classification
1022                         addParentTaxon(taxon, state, preferredFlag, state.getClassification());
1023                     }
1024                 }
1025             }
1026
1027
1028             private boolean existsInClassification(Taxon taxon, Classification classification, STATE state){
1029                 boolean exist = false;
1030                 ICdmRepository cdmAppController = state.getConfig().getCdmAppController();
1031             if(cdmAppController==null){
1032                 cdmAppController = this;
1033             }
1034             if (classification != null){
1035                 if (!taxon.getTaxonNodes().isEmpty()){
1036                     for (TaxonNode node:taxon.getTaxonNodes()){
1037                         if (node.getClassification().equals(classification)){
1038                             return true;
1039                         }
1040                     }
1041                 }
1042 // we do not need this because we already searched for taxa in db in the previous steps
1043 //              List<UuidAndTitleCache<TaxonNode>> uuidAndTitleCacheOfAllTaxa = cdmAppController.getClassificationService().getTaxonNodeUuidAndTitleCacheOfAcceptedTaxaByClassification(classification.getUuid());
1044 //              if (uuidAndTitleCacheOfAllTaxa != null){
1045 //                      for (UuidAndTitleCache p : uuidAndTitleCacheOfAllTaxa){
1046 //                          try{
1047 //                              if(p.getTitleCache().equals(taxon.getTitleCache())) {
1048 //                                  exist = true;
1049 //                              }
1050 //                          }
1051 //                          catch(Exception e){
1052 //                              logger.warn("TaxonNode doesn't seem to have a taxon");
1053 //                          }
1054 //                      }
1055 //              }
1056             }
1057                 return exist;
1058             }
1059
1060             /**
1061              * join DeterminationEvent to the Taxon Object
1062              * @param state : the ABCD import state
1063              * @param taxon: the current Taxon
1064              * @param preferredFlag :if the current name is preferred
1065              * @param derivedFacade : the derived Unit Facade
1066              */
1067             @SuppressWarnings("rawtypes")
1068         protected void linkDeterminationEvent(STATE state, Taxon taxon, boolean preferredFlag,  DerivedUnitFacade derivedFacade, String identifierStr, String dateStr) {
1069                 SpecimenImportConfiguratorBase config = state.getConfig();
1070                 if (logger.isDebugEnabled()){
1071                     logger.info("start linkdetermination with taxon:" + taxon.getUuid()+", "+taxon);
1072                 }
1073
1074                 DeterminationEvent determinationEvent = DeterminationEvent.NewInstance();
1075                 determinationEvent.setTaxonName(taxon.getName());
1076                 determinationEvent.setPreferredFlag(preferredFlag);
1077
1078
1079                 determinationEvent.setIdentifiedUnit(state.getDerivedUnitBase());
1080                 if (state.getPersonStore().get(identifierStr) != null){
1081                     determinationEvent.setActor((AgentBase)state.getPersonStore().get(identifierStr));
1082                 } else if (identifierStr != null){
1083                     Person identifier = Person.NewTitledInstance(identifierStr);
1084                     determinationEvent.setActor(identifier);
1085                 }
1086                 if (dateStr != null){
1087                     determinationEvent.setTimeperiod(TimePeriodParser.parseString(dateStr));
1088                 }
1089                 state.getDerivedUnitBase().addDetermination(determinationEvent);
1090
1091                 if (logger.isDebugEnabled()){
1092                     logger.debug("NB TYPES INFO: "+ state.getDataHolder().getStatusList().size());
1093                 }
1094                 for (SpecimenTypeDesignationStatus specimenTypeDesignationstatus : state.getDataHolder().getStatusList()) {
1095                     if (specimenTypeDesignationstatus != null) {
1096                         if (logger.isDebugEnabled()){
1097                             logger.debug("specimenTypeDesignationstatus :"+ specimenTypeDesignationstatus);
1098                         }
1099
1100                         ICdmRepository cdmAppController = config.getCdmAppController();
1101                         if(cdmAppController == null){
1102                             cdmAppController = this;
1103                         }
1104                         specimenTypeDesignationstatus = (SpecimenTypeDesignationStatus) cdmAppController.getTermService().find(specimenTypeDesignationstatus.getUuid());
1105                         //Designation
1106                         TaxonNameBase<?,?> name = taxon.getName();
1107                         SpecimenTypeDesignation designation = SpecimenTypeDesignation.NewInstance();
1108
1109                         designation.setTypeStatus(specimenTypeDesignationstatus);
1110                         designation.setTypeSpecimen(state.getDerivedUnitBase());
1111                         name.addTypeDesignation(designation, true);
1112                     }
1113                 }
1114
1115                 for (String[] fullReference : state.getDataHolder().getReferenceList()) {
1116
1117
1118                     String strReference=fullReference[0];
1119                     String citationDetail = fullReference[1];
1120                     String citationURL = fullReference[2];
1121                     List<Reference> references = getReferenceService().listByTitle(Reference.class, "strReference", MatchMode.EXACT, null, null, null, null, null);
1122
1123                     if (!references.isEmpty()){
1124                         Reference reference = null;
1125                         for (Reference refe: references) {
1126                             if (refe.getTitleCache().equalsIgnoreCase(strReference)) {
1127                                 reference =refe;
1128                                 break;
1129                             }
1130                         }
1131                         if (reference ==null){
1132                             reference = ReferenceFactory.newGeneric();
1133                             reference.setTitleCache(strReference, true);
1134                             save(reference, state);
1135                         }
1136                         determinationEvent.addReference(reference);
1137                     }
1138                 }
1139                 save(state.getDerivedUnitBase(), state);
1140
1141                 if (config.isAddIndividualsAssociationsSuchAsSpecimenAndObservations() && preferredFlag) {
1142                     //do not add IndividualsAssociation to non-preferred taxa
1143                     if (logger.isDebugEnabled()){
1144                         logger.debug("isDoCreateIndividualsAssociations");
1145                     }
1146
1147                     makeIndividualsAssociation(state, taxon, determinationEvent);
1148
1149                     save(state.getDerivedUnitBase(), state);
1150                 }
1151             }
1152
1153             /**
1154              * create and link each association (specimen, observation..) to the accepted taxon
1155              * @param state : the ABCD import state
1156              * @param taxon: the current Taxon
1157              * @param determinationEvent:the determinationevent
1158              */
1159             protected void makeIndividualsAssociation(STATE state, Taxon taxon, DeterminationEvent determinationEvent) {
1160                 SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
1161                 SpecimenUserInteraction sui = config.getSpecimenUserInteraction();
1162
1163                 if (logger.isDebugEnabled()){
1164                     logger.info("MAKE INDIVIDUALS ASSOCIATION");
1165                 }
1166
1167                 TaxonDescription taxonDescription = null;
1168                 Set<TaxonDescription> descriptions= taxon.getDescriptions();
1169 //              if (((Abcd206ImportConfigurator) state.getConfig()).isInteractWithUser()){
1170 //                  if(!state.isDescriptionGroupSet()){
1171 //                      taxonDescription = sui.askForDescriptionGroup(descriptions);
1172 //                      state.setDescriptionGroup(taxonDescription);
1173 //                      state.setDescriptionGroupSet(true);
1174 //                  }else{
1175 //                      taxonDescription=state.getDescriptionGroup();
1176 //                  }
1177 //              } else {
1178                     for (TaxonDescription description : descriptions){
1179                         Set<IdentifiableSource> sources =  new HashSet<>();
1180                         sources.addAll(description.getTaxon().getSources());
1181                         sources.addAll(description.getSources());
1182                         for (IdentifiableSource source:sources){
1183                             if(state.getRef().equals(source.getCitation())) {
1184                                 taxonDescription = description;
1185                             }
1186                         }
1187                     }
1188               //  }
1189                 if (taxonDescription == null){
1190                     taxonDescription = TaxonDescription.NewInstance(taxon, false);
1191                     if(sourceNotLinkedToElement(taxonDescription,state.getRef(),null)) {
1192                         taxonDescription.addSource(OriginalSourceType.Import, null, null, state.getRef(), null);
1193                     }
1194                     state.setDescriptionGroup(taxonDescription);
1195                     taxon.addDescription(taxonDescription);
1196                 }
1197
1198                 //PREPARE REFERENCE QUESTIONS
1199
1200                 Map<String,OriginalSourceBase<?>> sourceMap = new HashMap<String, OriginalSourceBase<?>>();
1201
1202                 List<IdentifiableSource> issTmp = new ArrayList<>();//getCommonService().list(IdentifiableSource.class, null, null, null, null);
1203                 List<DescriptionElementSource> issTmp2 = new ArrayList<>();//getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1204
1205                 Set<OriginalSourceBase> osbSet = new HashSet<OriginalSourceBase>();
1206                 if(issTmp2!=null) {
1207                     osbSet.addAll(issTmp2);
1208                 }
1209                 if(issTmp!=null) {
1210                     osbSet.addAll(issTmp);
1211                 }
1212
1213
1214                 addToSourceMap(sourceMap, osbSet);
1215
1216 //              if (((Abcd206ImportConfigurator) state.getConfig()).isInteractWithUser()){
1217 //                  List<OriginalSourceBase<?>> res = null;
1218 //                  if(!state.isDescriptionSourcesSet()){
1219 //                      res = sui.askForSource(sourceMap, "the description group ("+taxon+")",
1220 //                              "The current reference is "+state.getRef().getTitleCache(),getReferenceService(), state.getDataHolder().docSources);
1221 //                      state.setDescriptionRefs(res);
1222 //                      state.setDescriptionSourcesSet(true);
1223 //                  }
1224 //                  else{
1225 //                      res=state.getDescriptionRefs();
1226 //                  }
1227 //                  if(res !=null) {
1228 //                      for (OriginalSourceBase<?> sour:res){
1229 //                          if(sour.isInstanceOf(IdentifiableSource.class)){
1230 //                              try {
1231 //                                  if(sourceNotLinkedToElement(taxonDescription,sour)) {
1232 //                                      taxonDescription.addSource((IdentifiableSource)sour.clone());
1233 //                                  }
1234 //                              } catch (CloneNotSupportedException e) {
1235 //                                  logger.warn("no cloning?");
1236 //                              }
1237 //                          }else{
1238 //                              if(sourceNotLinkedToElement(taxonDescription,sour)) {
1239 //                                  taxonDescription.addSource(OriginalSourceType.Import,null, null, sour.getCitation(),sour.getCitationMicroReference());
1240 //                              }
1241 //                          }
1242 //                      }
1243 //                  }
1244 //              }
1245 //              else {
1246                     if(sourceNotLinkedToElement(taxonDescription,state.getRef(),null)) {
1247                         taxonDescription.addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1248                     }
1249 //              }
1250                 state.setDescriptionGroup(taxonDescription);
1251
1252                 IndividualsAssociation indAssociation = IndividualsAssociation.NewInstance();
1253                 Feature feature = makeFeature(state.getDerivedUnitBase());
1254                 indAssociation.setAssociatedSpecimenOrObservation(state.getDerivedUnitBase());
1255                 indAssociation.setFeature(feature);
1256 //
1257 //              if (((Abcd206ImportConfigurator) state.getConfig()).isInteractWithUser()){
1258 //                  sourceMap = new HashMap<String, OriginalSourceBase<?>>();
1259 //
1260 //                  issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
1261 //                  issTmp2 = getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1262 //
1263 //                  osbSet = new HashSet<OriginalSourceBase>();
1264 //                  if(issTmp2!=null) {
1265 //                      osbSet.addAll(issTmp2);
1266 //                  }
1267 //                  if(issTmp!=null) {
1268 //                      osbSet.addAll(issTmp);
1269 //                  }
1270 //
1271 //
1272 //                  addToSourceMap(sourceMap, osbSet);
1273 //
1274 //                  List<OriginalSourceBase<?>> sources =null;
1275 //                  if(!state.isAssociationSourcesSet()) {
1276 //                      sources = sui.askForSource(sourceMap,  "descriptive element (association) ",taxon.toString(),
1277 //                              getReferenceService(),state.getDataHolder().getDocSources());
1278 //                      state.setAssociationRefs(sources);
1279 //                      state.setAssociationSourcesSet(true);
1280 //                  }
1281 //                  else{
1282 //                      sources=state.getAssociationRefs();
1283 //                  }
1284 //                  if(sources !=null) {
1285 //                      for (OriginalSourceBase<?> source: sources) {
1286 //                          if(source !=null) {
1287 //                              if(source.isInstanceOf(DescriptionElementSource.class)){
1288 //                                  try {
1289 //                                      if(sourceNotLinkedToElement(indAssociation,source)) {
1290 //                                          indAssociation.addSource((DescriptionElementSource)source.clone());
1291 //                                      }
1292 //                                  } catch (CloneNotSupportedException e) {
1293 //                                      logger.warn("clone forbidden?");
1294 //                                  }
1295 //                              }else{
1296 //                                  if(sourceNotLinkedToElement(indAssociation,source)) {
1297 //                                      indAssociation.addSource(OriginalSourceType.Import,null, null, source.getCitation(),source.getCitationMicroReference());
1298 //                                  }
1299 //                                  try {
1300 //                                      if(sourceNotLinkedToElement(state.getDerivedUnitBase(), source)) {
1301 //                                          state.getDerivedUnitBase().addSource((IdentifiableSource) source.clone());
1302 //                                      }
1303 //                                  } catch (CloneNotSupportedException e) {
1304 //                                      // TODO Auto-generated catch block
1305 //                                      e.printStackTrace();
1306 //                                  }
1307 //                              }
1308 //
1309 //                          }
1310 //                      }
1311 //                  }
1312 //              }else {
1313                     if(sourceNotLinkedToElement(indAssociation,state.getRef(),null)) {
1314                         indAssociation.addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1315                     }
1316                     if(sourceNotLinkedToElement(state.getDerivedUnitBase(), state.getRef(),null)) {
1317                         state.getDerivedUnitBase().addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1318                     }
1319                     for (Reference citation : determinationEvent.getReferences()) {
1320                         if(sourceNotLinkedToElement(indAssociation,citation,null))
1321                         {
1322                             indAssociation.addSource(DescriptionElementSource.NewInstance(OriginalSourceType.Import, null, null, citation, null));
1323                         }
1324                         if(sourceNotLinkedToElement(state.getDerivedUnitBase(), state.getRef(),null)) {
1325                             state.getDerivedUnitBase().addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1326                         }
1327                     }
1328            //     }
1329
1330                 taxonDescription.addElement(indAssociation);
1331
1332                 save(taxonDescription, state);
1333                 save(taxon, state);
1334                 state.getReport().addDerivate(state.getDerivedUnitBase(), config);
1335                 state.getReport().addIndividualAssociation(taxon, state.getDataHolder().getUnitID(), state.getDerivedUnitBase());
1336             }
1337
1338             /**
1339              * @param derivedUnitBase2
1340              * @param ref2
1341              * @param object
1342              * @return
1343              */
1344             private boolean sourceNotLinkedToElement(DerivedUnit derivedUnitBase2, Reference b, String d) {
1345                 Set<IdentifiableSource> linkedSources = derivedUnitBase2.getSources();
1346                 for (IdentifiableSource is:linkedSources){
1347                     Reference a = is.getCitation();
1348                     String c = is.getCitationMicroReference();
1349
1350                     boolean refMatch=false;
1351                     boolean microMatch=false;
1352
1353                     try{
1354                         if (a==null && b==null) {
1355                             refMatch=true;
1356                         }
1357                         if (a!=null && b!=null) {
1358                             if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1359                                 refMatch=true;
1360                             }
1361                         }
1362                     }catch(Exception e){}
1363
1364
1365                     try{
1366                         if (c==null && d==null) {
1367                             microMatch=true;
1368                         }
1369                         if(c!=null && d!=null) {
1370                             if(c.equalsIgnoreCase(d)) {
1371                                 microMatch=true;
1372                             }
1373                         }
1374                     }
1375                     catch(Exception e){}
1376
1377                     if (microMatch && refMatch) {
1378                         return false;
1379                     }
1380
1381
1382                 }
1383                 return true;
1384             }
1385
1386             private <T extends OriginalSourceBase<?>> boolean  sourceNotLinkedToElement(ISourceable<T> sourcable, Reference reference, String microReference) {
1387                 Set<T> linkedSources = sourcable.getSources();
1388                 for (T is:linkedSources){
1389                     Reference unitReference = is.getCitation();
1390                     String unitMicroReference = is.getCitationMicroReference();
1391
1392                     boolean refMatch=false;
1393                     boolean microMatch=false;
1394
1395                     try{
1396                         if (unitReference==null && reference==null) {
1397                             refMatch=true;
1398                         }
1399                         if (unitReference!=null && reference!=null) {
1400                             if (unitReference.getTitleCache().equalsIgnoreCase(reference.getTitleCache())) {
1401                                 refMatch=true;
1402                             }
1403                         }
1404                     }catch(Exception e){}
1405
1406                     try{
1407                         if (unitMicroReference==null && microReference==null) {
1408                             microMatch=true;
1409                         }
1410                         if(unitMicroReference!=null && microReference!=null) {
1411                             if(unitMicroReference.equalsIgnoreCase(microReference)) {
1412                                 microMatch=true;
1413                             }
1414                         }
1415                     }
1416                     catch(Exception e){}
1417
1418                     if (microMatch && refMatch) {
1419                         return false;
1420                     }
1421                 }
1422                 return true;
1423             }
1424
1425             /**
1426              * look for the Feature object (FieldObs, Specimen,...)
1427              * @param unit : a specimen or obersvation base
1428              * @return the corresponding Feature
1429              */
1430             private Feature makeFeature(SpecimenOrObservationBase<?> unit) {
1431                 SpecimenOrObservationType type = unit.getRecordBasis();
1432
1433
1434
1435                 if (type.isFeatureObservation()){
1436                     return Feature.OBSERVATION();
1437                 }else if (type.isFeatureSpecimen()){
1438                     return Feature.SPECIMEN();
1439                 }else if (type == SpecimenOrObservationType.DerivedUnit){
1440                     return Feature.OBSERVATION();
1441                     //            return getFeature("Specimen or observation");
1442                 }else{
1443                     String message = "Unhandled record basis '%s' for defining individuals association feature type. Use default.";
1444                     logger.warn(String.format(message, type.getMessage()));
1445                     return Feature.OBSERVATION();
1446                     //            return getFeature("Specimen or observation");
1447
1448                 }
1449             }
1450
1451
1452             /**
1453              * @param sourceMap
1454              * @param osbSet
1455              */
1456             protected void addToSourceMap(Map<String, OriginalSourceBase<?>> sourceMap, Set<OriginalSourceBase> osbSet) {
1457                 for( OriginalSourceBase<?> osb:osbSet) {
1458                     if(osb.getCitation()!=null && osb.getCitationMicroReference() !=null  && !osb.getCitationMicroReference().isEmpty()) {
1459                         try{
1460                             sourceMap.put(osb.getCitation().getTitleCache()+ "---"+osb.getCitationMicroReference(),osb);
1461                         }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1462                     } else if(osb.getCitation()!=null){
1463                         try{
1464                             sourceMap.put(osb.getCitation().getTitleCache(),osb);
1465                         }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1466                     }
1467                 }
1468             }
1469
1470         /**
1471          * @param state
1472          * @param itemObject
1473          * @param handleAssociatedUnits
1474          */
1475         public void handleSingleUnit(Abcd206ImportState state, Object itemObject, boolean handleAssociatedUnits) {
1476             // TODO Auto-generated method stub
1477
1478         }
1479
1480 }