cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/SpecimenImportBase.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.specimen;
  11
  12
  13 import java.util.ArrayList;
  14 import java.util.HashMap;
  15 import java.util.HashSet;
  16 import java.util.List;
  17 import java.util.Map;
  18 import java.util.Set;
  19 import java.util.UUID;
  20
  21 import org.apache.log4j.Logger;
  22
  23 import eu.etaxonomy.cdm.api.application.ICdmRepository;
  24 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
  25 import eu.etaxonomy.cdm.api.service.config.FindOccurrencesConfigurator;
  26 import eu.etaxonomy.cdm.api.service.pager.Pager;
  27 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  28 import eu.etaxonomy.cdm.io.common.CdmImportBase;
  29 import eu.etaxonomy.cdm.io.common.IImportConfigurator;
  30 import eu.etaxonomy.cdm.io.specimen.abcd206.in.Identification;
  31 import eu.etaxonomy.cdm.io.specimen.abcd206.in.SpecimenImportReport;
  32 import eu.etaxonomy.cdm.model.agent.AgentBase;
  33 import eu.etaxonomy.cdm.model.agent.Institution;
  34 import eu.etaxonomy.cdm.model.agent.Person;
  35 import eu.etaxonomy.cdm.model.agent.Team;
  36 import eu.etaxonomy.cdm.model.common.CdmBase;
  37 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  38 import eu.etaxonomy.cdm.model.common.LanguageString;
  39 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  40 import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
  41 import eu.etaxonomy.cdm.model.description.Feature;
  42 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  43 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  44 import eu.etaxonomy.cdm.model.name.INonViralName;
  45 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  46 import eu.etaxonomy.cdm.model.name.Rank;
  47 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
  48 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
  49 import eu.etaxonomy.cdm.model.name.TaxonName;
  50 import eu.etaxonomy.cdm.model.name.TaxonNameFactory;
  51 import eu.etaxonomy.cdm.model.occurrence.Collection;
  52 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  53 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
  54 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
  55 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationType;
  56 import eu.etaxonomy.cdm.model.reference.ISourceable;
  57 import eu.etaxonomy.cdm.model.reference.OriginalSourceBase;
  58 import eu.etaxonomy.cdm.model.reference.OriginalSourceType;
  59 import eu.etaxonomy.cdm.model.reference.Reference;
  60 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  61 import eu.etaxonomy.cdm.model.taxon.Classification;
  62 import eu.etaxonomy.cdm.model.taxon.Synonym;
  63 import eu.etaxonomy.cdm.model.taxon.Taxon;
  64 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  65 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  66 import eu.etaxonomy.cdm.model.term.DefinedTerm;
  67 import eu.etaxonomy.cdm.persistence.query.MatchMode;
  68 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
  69 import eu.etaxonomy.cdm.strategy.parser.ParserProblem;
  70 import eu.etaxonomy.cdm.strategy.parser.TimePeriodParser;
  71
  72
  73 /**
  74  * @author p.kelbert
  75  * @since 20.10.2008
  76  */
  77 public abstract class SpecimenImportBase<CONFIG extends IImportConfigurator, STATE extends SpecimenImportStateBase>
  78         extends CdmImportBase<CONFIG, STATE> {
  79
  80     private static final long serialVersionUID = 4423065367998125678L;
  81     private static final Logger logger = Logger.getLogger(SpecimenImportBase.class);
  82
  83         protected static final UUID SPECIMEN_SCAN_TERM = UUID.fromString("acda15be-c0e2-4ea8-8783-b9b0c4ad7f03");
  84
  85         private static final String COLON = ":";
  86
  87         protected Map<String, DefinedTerm> kindOfUnitsMap;
  88
  89
  90
  91         @Override
  92     protected abstract void doInvoke(STATE state);
  93
  94         /**
  95          * Handle a single unit
  96          * @param state
  97          * @param item
  98          */
  99         protected abstract void handleSingleUnit(STATE state, Object item) ;
 100
 101
 102
 103         protected TaxonName getOrCreateTaxonName(String scientificName, Rank rank, boolean preferredFlag, STATE state, int unitIndexInAbcdFile){
 104             TaxonName taxonName = null;
 105         SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
 106
 107         //check atomised name data for rank
 108         //new name will be created
 109         TaxonName atomisedTaxonName = null;
 110         if (rank==null && unitIndexInAbcdFile>=0 && ((state.getDataHolder().getAtomisedIdentificationList() != null && !state.getDataHolder().getAtomisedIdentificationList().isEmpty())|| state.getDataHolder().getAtomisedIdentificationList().size() > 0)) {
 111             atomisedTaxonName = setTaxonNameByType(state.getDataHolder().getAtomisedIdentificationList().get(unitIndexInAbcdFile), scientificName, state);
 112             if(atomisedTaxonName!=null){
 113                 rank = atomisedTaxonName.getRank();
 114             }
 115         }
 116         if(config.isReuseExistingTaxaWhenPossible()){
 117             TaxonName parsedName = atomisedTaxonName;
 118             if(parsedName==null){
 119
 120                 parsedName = parseScientificName(scientificName, state, state.getReport(), rank);
 121
 122             }
 123             atomisedTaxonName = parsedName;
 124             if(config.isIgnoreAuthorship() && parsedName!=null){// && preferredFlag){
 125                 // do not ignore authorship for non-preferred names because they need
 126                 // to be created for the determination history
 127                 String nameCache = TaxonName.castAndDeproxy(parsedName).getNameCache();
 128                 List<TaxonName> names = getNameService().findNamesByNameCache(nameCache, MatchMode.EXACT, null);
 129                 if (!names.isEmpty()){
 130                      taxonName = getBestMatchingName(scientificName, new ArrayList<>(names), state);
 131                 }
 132                 if (taxonName == null && !names.isEmpty()){
 133                     taxonName = names.get(0);
 134                 }
 135
 136             } else {
 137                 //search for existing names
 138                 List<TaxonName> names = getNameService().listByTitleWithRestrictions(TaxonName.class, scientificName, MatchMode.EXACT, null, null, null, null, null);
 139                 taxonName = getBestMatchingName(scientificName, names, state);
 140                 //still nothing found -> try with the atomised name full title cache
 141                 if(taxonName==null && atomisedTaxonName!=null){
 142                     names = getNameService().listByTitleWithRestrictions(TaxonName.class, atomisedTaxonName.getFullTitleCache(), MatchMode.EXACT, null, null, null, null, null);
 143                     taxonName = getBestMatchingName(atomisedTaxonName.getTitleCache(), names, state);
 144                     //still nothing found -> try with the atomised name title cache
 145                     if(taxonName==null){
 146                         names = getNameService().listByTitleWithRestrictions(TaxonName.class, atomisedTaxonName.getTitleCache(), MatchMode.EXACT, null, null, null, null, null);
 147                         taxonName = getBestMatchingName(atomisedTaxonName.getTitleCache(), names, state);
 148                     }
 149                 }
 150
 151             }
 152
 153         }
 154
 155         if(taxonName==null && atomisedTaxonName!=null){
 156             taxonName = atomisedTaxonName;
 157             state.getReport().addName(taxonName);
 158             logger.info("Created new taxon name "+taxonName);
 159             if(taxonName.hasProblem()){
 160                 state.getReport().addInfoMessage(String.format("Created %s with parsing problems", taxonName));
 161             }
 162             if(!atomisedTaxonName.getTitleCache().equals(scientificName)){
 163                 state.getReport().addInfoMessage(String.format("Taxon %s was parsed as %s", scientificName, atomisedTaxonName.getTitleCache()));
 164             }
 165         }
 166         else if(taxonName==null){
 167             //create new taxon name
 168
 169             if (state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNAFP)){
 170                 taxonName = TaxonNameFactory.NewBotanicalInstance(rank);
 171             }else if (state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICZN)){
 172                 taxonName = TaxonNameFactory.NewZoologicalInstance(rank);
 173             }else{
 174                 taxonName = TaxonNameFactory.NewNonViralInstance(rank);
 175             }
 176             taxonName.setFullTitleCache(scientificName,true);
 177             taxonName.setTitleCache(scientificName, true);
 178             state.getReport().addName(taxonName);
 179             logger.info("Created new taxon name "+taxonName);
 180         }
 181         if (taxonName != null){
 182             state.names.put(taxonName.getNameCache(), taxonName);
 183         }
 184         if(!taxonName.isPersited()) {
 185             save(taxonName, state);
 186         }
 187         return taxonName;
 188     }
 189
 190          protected TaxonName getBestMatchingName(String scientificName, java.util.Collection<TaxonName> names, STATE state){
 191                 Set<TaxonName> namesWithAcceptedTaxa = new HashSet<>();
 192                 List<TaxonName> namesWithAcceptedTaxaInClassification = new ArrayList<>();
 193                 for (TaxonName name : names) {
 194                     if(!name.getTaxa().isEmpty()){
 195                         Set<Taxon> taxa = name.getTaxa();
 196                         for (Taxon taxon:taxa){
 197                             if (!taxon.getTaxonNodes().isEmpty()){
 198                                 //use only taxa included in a classification
 199                                 for (TaxonNode node:taxon.getTaxonNodes()){
 200                                     if (state.getClassification() != null && node.getClassification().equals(state.getClassification())){
 201                                         namesWithAcceptedTaxaInClassification.add(name);
 202                                     }else {
 203                                         namesWithAcceptedTaxa.add(name);
 204                                     }
 205                                 }
 206
 207                             }
 208                         }
 209
 210                     }
 211                 }
 212                 String message = String.format("More than one taxon name was found for %s, maybe in other classifications!", scientificName);
 213                 //check for names with accepted taxa in classification
 214                 if(namesWithAcceptedTaxaInClassification.size()>0){
 215                 if(namesWithAcceptedTaxaInClassification.size()>1){
 216
 217                     state.getReport().addInfoMessage(message);
 218                     logger.warn(message);
 219                     return null;
 220                 }
 221                 return namesWithAcceptedTaxaInClassification.iterator().next();
 222             }
 223               //check for any names with accepted taxa
 224                 if(namesWithAcceptedTaxa.size()>0){
 225                     if(namesWithAcceptedTaxa.size()>1){
 226
 227                         state.getReport().addInfoMessage(message);
 228                         logger.warn(message);
 229                         return null;
 230                     }
 231                     return namesWithAcceptedTaxa.iterator().next();
 232                 }
 233                 //no names with accepted taxa found -> check accepted taxa of synonyms
 234                 List<Taxon> taxaFromSynonyms = new ArrayList<>();
 235                 for (TaxonName name : names) {
 236                     Set<TaxonBase> taxonBases = name.getTaxonBases();
 237                     for (TaxonBase taxonBase : taxonBases) {
 238                         if(taxonBase.isInstanceOf(Synonym.class)){
 239                             Synonym synonym = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
 240                             taxaFromSynonyms.add(synonym.getAcceptedTaxon());
 241                         }
 242                     }
 243                 }
 244                 if(taxaFromSynonyms.size()>0){
 245                     if(taxaFromSynonyms.size()>1){
 246                         state.getReport().addInfoMessage(message);
 247                         logger.warn(message);
 248                         return null;
 249                     }
 250                     return taxaFromSynonyms.iterator().next().getName();
 251                 }
 252                 //no accepted and no synonyms -> return one of the names and create a new taxon
 253                 if (names.isEmpty()){
 254                     return null;
 255                 }else{
 256                     return names.iterator().next();
 257                 }
 258             }
 259          /**
 260              * Parse automatically the scientific name
 261              * @param scientificName the scientific name to parse
 262              * @param state the current import state
 263              * @param report the import report
 264              * @return a parsed name
 265              */
 266
 267             protected TaxonName parseScientificName(String scientificName, STATE state, SpecimenImportReport report, Rank rank) {
 268
 269                 NonViralNameParserImpl nvnpi = NonViralNameParserImpl.NewInstance();
 270                 TaxonName taxonName = null;
 271                 boolean problem = false;
 272
 273                 if (logger.isDebugEnabled()){
 274                     logger.debug("parseScientificName " + state.getDataHolder().getNomenclatureCode().toString());
 275                 }
 276
 277                 if (state.getDataHolder().getNomenclatureCode() != null && (state.getDataHolder().getNomenclatureCode().toString().equals("Zoological") || state.getDataHolder().getNomenclatureCode().toString().contains("ICZN"))) {
 278                     taxonName = (TaxonName)nvnpi.parseFullName(scientificName, NomenclaturalCode.ICZN, rank);
 279                     if (taxonName.hasProblem()) {
 280                         problem = true;
 281                     }
 282                 }
 283                 else if (state.getDataHolder().getNomenclatureCode() != null && (state.getDataHolder().getNomenclatureCode().toString().equals("Botanical") || state.getDataHolder().getNomenclatureCode().toString().contains("ICBN")  || state.getDataHolder().getNomenclatureCode().toString().contains("ICNAFP"))) {
 284                     taxonName = (TaxonName)nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNAFP, rank);
 285                     if (taxonName.hasProblem()) {
 286                         problem = true;
 287                     }
 288                 }
 289                 else if (state.getDataHolder().getNomenclatureCode() != null && (state.getDataHolder().getNomenclatureCode().toString().equals("Bacterial") || state.getDataHolder().getNomenclatureCode().toString().contains("ICBN"))) {
 290                     taxonName = (TaxonName)nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNB, rank);
 291                     if (taxonName.hasProblem()) {
 292                         problem = true;
 293                     }
 294                 }
 295                 else if (state.getDataHolder().getNomenclatureCode() != null && (state.getDataHolder().getNomenclatureCode().toString().equals("Cultivar") || state.getDataHolder().getNomenclatureCode().toString().contains("ICNCP"))) {
 296                     taxonName = (TaxonName)nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNCP, rank);
 297                     if (taxonName.hasProblem()) {
 298                         problem = true;
 299                     }
 300                 }
 301                 if (problem) {
 302                     String message = String.format("Parsing problems for %s", scientificName);
 303                     if(taxonName!=null){
 304                         for (ParserProblem parserProblem : taxonName.getParsingProblems()) {
 305                             message += "\n\t- "+parserProblem;
 306                         }
 307                     }
 308                     report.addInfoMessage(message);
 309                     logger.info(message);
 310                 }
 311                 return taxonName;
 312
 313             }
 314
 315             /**
 316              * Create the name without automatic parsing, either because it failed, or because the user deactivated it.
 317              * The name is built upon the ABCD fields
 318              * @param atomisedMap : the ABCD atomised fields
 319              * @param fullName : the full scientific name
 320              * @param state
 321              * @return the corresponding Botanical or Zoological or... name
 322              */
 323             protected TaxonName setTaxonNameByType(
 324                     HashMap<String, String> atomisedMap, String fullName, STATE state) {
 325                 boolean problem = false;
 326                 if (logger.isDebugEnabled()){
 327                     logger.debug("settaxonnamebytype " + state.getDataHolder().getNomenclatureCode().toString());
 328                 }
 329
 330                 if (state.getDataHolder().getNomenclatureCode().equals("Zoological") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICZN.getUuid())) {
 331                     TaxonName taxonName = TaxonNameFactory.NewZoologicalInstance(null);
 332                     taxonName.setFullTitleCache(fullName, true);
 333                     taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
 334                     taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
 335                     taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap,"SpeciesEpithet")));
 336                     taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap,"SubspeciesEpithet")));
 337
 338                     if (taxonName.getGenusOrUninomial() != null){
 339                         taxonName.setRank(Rank.GENUS());
 340                     }
 341
 342                     if (taxonName.getInfraGenericEpithet() != null){
 343                         taxonName.setRank(Rank.SUBGENUS());
 344                     }
 345
 346                     if (taxonName.getSpecificEpithet() != null){
 347                         taxonName.setRank(Rank.SPECIES());
 348                     }
 349
 350                     if (taxonName.getInfraSpecificEpithet() != null){
 351                         taxonName.setRank(Rank.SUBSPECIES());
 352                     }
 353
 354                     Team team = null;
 355                     if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 356                         team = Team.NewInstance();
 357                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
 358                     }
 359                     else {
 360                         if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 361                             team = Team.NewInstance();
 362                             team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
 363                         }
 364                     }
 365                     if (team != null) {
 366                         taxonName.setBasionymAuthorship(team);
 367                     }
 368                     else {
 369                         if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 370                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
 371                         }
 372                         else if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 373                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamAndYear"));
 374                         }
 375                     }
 376                     if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
 377                         team = Team.NewInstance();
 378                         team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
 379                         taxonName.setCombinationAuthorship(team);
 380                     }
 381                     if (taxonName.hasProblem()) {
 382                         logger.info("pb ICZN");
 383                         problem = true;
 384                     }
 385                     else {
 386                         return taxonName;
 387                     }
 388                 }
 389                 else if (state.getDataHolder().getNomenclatureCode().equals("Botanical") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNAFP.getUuid())) {
 390                     TaxonName taxonName = parseScientificName(fullName, state, state.getReport(), null);
 391                     if (taxonName != null){
 392                         return taxonName;
 393                     }
 394                     else{
 395                         taxonName = TaxonNameFactory.NewBotanicalInstance(null);
 396                     }
 397                     taxonName.setFullTitleCache(fullName, true);
 398                     taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
 399                     taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "FirstEpithet")));
 400                     taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "InfraSpeEpithet")));
 401                     try {
 402                         taxonName.setRank(Rank.getRankByName(getFromMap(atomisedMap, "Rank")));
 403                     } catch (Exception e) {
 404                         if (taxonName.getInfraSpecificEpithet() != null){
 405                             taxonName.setRank(Rank.SUBSPECIES());
 406                         }
 407                         else if (taxonName.getSpecificEpithet() != null){
 408                             taxonName.setRank(Rank.SPECIES());
 409                         }
 410                         else if (taxonName.getInfraGenericEpithet() != null){
 411                             taxonName.setRank(Rank.SUBGENUS());
 412                         }
 413                         else if (taxonName.getGenusOrUninomial() != null){
 414                             taxonName.setRank(Rank.GENUS());
 415                         }
 416                     }
 417                     Team team = null;
 418                     if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 419                         team = Team.NewInstance();
 420                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
 421                         taxonName.setBasionymAuthorship(team);
 422                     }
 423                     if (getFromMap(atomisedMap, "AuthorTeam") != null) {
 424                         team = Team.NewInstance();
 425                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeam"), true);
 426                         taxonName.setCombinationAuthorship(team);
 427                     }
 428                     if (team == null) {
 429                         if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 430                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
 431                         }
 432                         else if (getFromMap(atomisedMap, "AuthorTeam") != null) {
 433                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeam"));
 434                         }
 435                     }
 436                     if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
 437                         team = Team.NewInstance();
 438                         team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
 439                         taxonName.setCombinationAuthorship(team);
 440                     }
 441                     if (taxonName.hasProblem()) {
 442                         logger.info("pb ICBN");
 443                         problem = true;
 444                     }
 445                     else {
 446                         return taxonName;
 447                     }
 448                 }
 449                 else if (state.getDataHolder().getNomenclatureCode().equals("Bacterial") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNB.getUuid())) {
 450                     TaxonName taxonName = TaxonNameFactory.NewBacterialInstance(null);
 451                     taxonName.setFullTitleCache(fullName, true);
 452                     taxonName.setGenusOrUninomial(getFromMap(atomisedMap, "Genus"));
 453                     taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
 454                     taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "Species")));
 455                     taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "SubspeciesEpithet")));
 456
 457                     if (taxonName.getGenusOrUninomial() != null){
 458                         taxonName.setRank(Rank.GENUS());
 459                     }
 460                     else if (taxonName.getInfraGenericEpithet() != null){
 461                         taxonName.setRank(Rank.SUBGENUS());
 462                     }
 463                     else if (taxonName.getSpecificEpithet() != null){
 464                         taxonName.setRank(Rank.SPECIES());
 465                     }
 466                     else if (taxonName.getInfraSpecificEpithet() != null){
 467                         taxonName.setRank(Rank.SUBSPECIES());
 468                     }
 469
 470                     if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 471                         Team team = Team.NewInstance();
 472                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
 473                         taxonName.setCombinationAuthorship(team);
 474                     }
 475                     if (getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear") != null) {
 476                         Team team = Team.NewInstance();
 477                         team.setTitleCache(getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear"), true);
 478                         taxonName.setBasionymAuthorship(team);
 479                     }
 480                     if (taxonName.hasProblem()) {
 481                         logger.info("pb ICNB");
 482                         problem = true;
 483                     }
 484                     else {
 485                         return taxonName;
 486                     }
 487                 }
 488                 else if (state.getDataHolder().getNomenclatureCode().equals("Cultivar")) {
 489                     TaxonName taxonName = TaxonNameFactory.NewCultivarInstance(null);
 490
 491                     if (taxonName.hasProblem()) {
 492                         logger.info("pb ICNCP");
 493                         problem = true;
 494                     }
 495                     else {
 496                         return taxonName;
 497                     }
 498                     return taxonName;
 499                 }
 500
 501                 if (problem) {
 502                     logger.info("Problem im setTaxonNameByType ");
 503                     TaxonName taxonName = TaxonNameFactory.NewNonViralInstance(null);
 504                     taxonName.setFullTitleCache(fullName, true);
 505                     return taxonName;
 506                 }
 507                 TaxonName tn = TaxonNameFactory.NewNonViralInstance(null);
 508                 return tn;
 509             }
 510
 511             /**
 512              * Get a formated string from a hashmap
 513              * @param atomisedMap
 514              * @param key
 515              * @return
 516              */
 517             private String getFromMap(HashMap<String, String> atomisedMap, String key) {
 518                 String value = null;
 519                 if (atomisedMap.containsKey(key)) {
 520                     value = atomisedMap.get(key);
 521                 }
 522
 523                 try {
 524                     if (value != null && key.matches(".*Year.*")) {
 525                         value = value.trim();
 526                         if (value.matches("[a-z A-Z ]*[0-9]{4}$")) {
 527                             String tmp = value.split("[0-9]{4}$")[0];
 528                             int year = Integer.parseInt(value.split(tmp)[1]);
 529                             if (year >= 1752) {
 530                                 value = tmp;
 531                             }
 532                             else {
 533                                 value = null;
 534                             }
 535                         }
 536                         else {
 537                             value = null;
 538                         }
 539                     }
 540                 }
 541                 catch (Exception e) {
 542                     value = null;
 543                 }
 544                 return value;
 545             }
 546
 547             /**
 548              * Very fast and dirty implementation to allow handling of transient objects as described in
 549              * https://dev.e-taxonomy.eu/trac/ticket/3726
 550              *
 551              * Not yet complete.
 552              *
 553              * @param cdmBase
 554              * @param state
 555              */
 556             protected UUID save(CdmBase cdmBase, SpecimenImportStateBase state) {
 557                 ICdmRepository cdmRepository = state.getConfig().getCdmAppController();
 558                 if (cdmRepository == null){
 559                     cdmRepository = this;
 560                 }
 561
 562                 if (cdmBase.isInstanceOf(LanguageString.class)){
 563                     return cdmRepository.getTermService().saveLanguageData(CdmBase.deproxy(cdmBase, LanguageString.class));
 564                 }else if (cdmBase.isInstanceOf(SpecimenOrObservationBase.class)){
 565                     SpecimenOrObservationBase specimen = CdmBase.deproxy(cdmBase, SpecimenOrObservationBase.class);
 566
 567                     return cdmRepository.getOccurrenceService().saveOrUpdate(specimen);
 568                 }else if (cdmBase.isInstanceOf(Reference.class)){
 569                     return cdmRepository.getReferenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, Reference.class));
 570                 }else if (cdmBase.isInstanceOf(Classification.class)){
 571                     return cdmRepository.getClassificationService().saveOrUpdate(CdmBase.deproxy(cdmBase, Classification.class));
 572                 }else if (cdmBase.isInstanceOf(AgentBase.class)){
 573                     return cdmRepository.getAgentService().saveOrUpdate(CdmBase.deproxy(cdmBase, AgentBase.class));
 574                 }else if (cdmBase.isInstanceOf(Collection.class)){
 575                     return cdmRepository.getCollectionService().saveOrUpdate(CdmBase.deproxy(cdmBase, Collection.class));
 576                 }else if (cdmBase.isInstanceOf(DescriptionBase.class)){
 577                     DescriptionBase description = CdmBase.deproxy(cdmBase, DescriptionBase.class);
 578
 579                     return cdmRepository.getDescriptionService().saveOrUpdate(description);
 580                 }else if (cdmBase.isInstanceOf(TaxonBase.class)){
 581                     return cdmRepository.getTaxonService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonBase.class));
 582                 }else if (cdmBase.isInstanceOf(TaxonName.class)){
 583                     return cdmRepository.getNameService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonName.class));
 584                 }else if (cdmBase.isInstanceOf(TaxonNode.class)){
 585                 return cdmRepository.getTaxonNodeService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonNode.class));
 586             }else{
 587                     throw new IllegalArgumentException("Class not supported in save method: " + CdmBase.deproxy(cdmBase, CdmBase.class).getClass().getSimpleName());
 588                 }
 589
 590             }
 591
 592
 593             protected SpecimenOrObservationBase findExistingSpecimen(String unitId, SpecimenImportStateBase state){
 594                 ICdmRepository cdmAppController = state.getConfig().getCdmAppController();
 595                 if(cdmAppController==null){
 596                     cdmAppController = this;
 597                 }
 598                 FindOccurrencesConfigurator config = new FindOccurrencesConfigurator();
 599                 config.setSignificantIdentifier(unitId);
 600                 List<String> propertyPaths = new ArrayList<>();
 601                 propertyPaths.add("derivedFrom.*");
 602                 config.setPropertyPaths(propertyPaths);
 603                 commitTransaction(state.getTx());
 604                 state.setTx(startTransaction());
 605                 try{
 606                         Pager<SpecimenOrObservationBase> existingSpecimens = cdmAppController.getOccurrenceService().findByTitle(config);
 607                         if(!existingSpecimens.getRecords().isEmpty()){
 608                             if(existingSpecimens.getRecords().size()==1){
 609                                 return existingSpecimens.getRecords().iterator().next();
 610                             }
 611                         }
 612
 613                 }catch(NullPointerException e){
 614                         logger.error("searching for existing specimen creates NPE: " + config.getSignificantIdentifier());
 615                         e.printStackTrace();
 616                 }
 617
 618
 619                 return null;
 620             }
 621
 622             protected abstract void importAssociatedUnits(STATE state, Object item, DerivedUnitFacade derivedUnitFacade);
 623
 624             /**
 625              * getFacade : get the DerivedUnitFacade based on the recordBasis
 626              * @param state
 627              *
 628              * @return DerivedUnitFacade
 629              */
 630             protected DerivedUnitFacade getFacade(STATE state) {
 631                 if (logger.isDebugEnabled()){
 632                     logger.info("getFacade()");
 633                 }
 634                 SpecimenOrObservationType type = null;
 635
 636                 // create specimen
 637                 if (NB((state.getDataHolder().getRecordBasis())) != null) {
 638                     if (state.getDataHolder().getRecordBasis().toLowerCase().startsWith("s") || state.getDataHolder().getRecordBasis().toLowerCase().indexOf("specimen")>-1) {// specimen
 639                         type = SpecimenOrObservationType.PreservedSpecimen;
 640                     }
 641                     if (state.getDataHolder().getRecordBasis().toLowerCase().startsWith("o") ||state.getDataHolder().getRecordBasis().toLowerCase().indexOf("observation")>-1 ) {
 642                         type = SpecimenOrObservationType.Observation;
 643                     }
 644                     if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("fossil")>-1){
 645                         type = SpecimenOrObservationType.Fossil;
 646                     }
 647                     if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("living")>-1) {
 648                         type = SpecimenOrObservationType.LivingSpecimen;
 649                     }
 650                     if (type == null) {
 651                         logger.info("The basis of record does not seem to be known: " + state.getDataHolder().getRecordBasis());
 652                         type = SpecimenOrObservationType.DerivedUnit;
 653                     }
 654                     // TODO fossils?
 655                 } else {
 656                     logger.info("The basis of record is null");
 657                     type = SpecimenOrObservationType.DerivedUnit;
 658                 }
 659                 DerivedUnitFacade derivedUnitFacade = DerivedUnitFacade.NewInstance(type);
 660                 return derivedUnitFacade;
 661             }
 662
 663             /**
 664              * Look if the Institution does already exist
 665              * @param institutionCode: a string with the institutioncode
 666              * @param config : the configurator
 667              * @return the Institution (existing or new)
 668              */
 669             protected Institution getInstitution(String institutionCode, STATE state) {
 670                 SpecimenImportConfiguratorBase config = state.getConfig();
 671                 Institution institution=null;
 672                 institution = (Institution)state.institutions.get(institutionCode);
 673                 if (institution != null){
 674                     return institution;
 675                 }
 676                 List<Institution> institutions;
 677                 try {
 678                     institutions = getAgentService().searchInstitutionByCode(institutionCode);
 679
 680                 } catch (Exception e) {
 681                     institutions = new ArrayList<Institution>();
 682                     logger.warn(e);
 683                 }
 684                 if (institutions.size() > 0 && config.isReuseExistingMetaData()) {
 685                     for (Institution institut:institutions){
 686                         try{
 687                             if (institut.getCode().equalsIgnoreCase(institutionCode)) {
 688                                 institution=institut;
 689                                 break;
 690                             }
 691                         }catch(Exception e){logger.warn("no institution code in the db");}
 692                     }
 693                 }
 694                 if (logger.isDebugEnabled()){
 695                     if(institution !=null) {
 696                         logger.info("getinstitution " + institution.toString());
 697                     }
 698                 }
 699                 if (institution == null){
 700                     // create institution
 701                     institution = Institution.NewInstance();
 702                     institution.setCode(institutionCode);
 703                     institution.setTitleCache(institutionCode, true);
 704                     UUID uuid = save(institution, state);
 705                 }
 706
 707
 708                 state.institutions.put(institutionCode, institution);
 709                 return institution;
 710             }
 711
 712             /**
 713              * Look if the Collection does already exist
 714              * @param collectionCode
 715              * @param collectionCode: a string
 716              * @param config : the configurator
 717              * @return the Collection (existing or new)
 718              */
 719             protected Collection getCollection(Institution institution, String collectionCode, STATE state) {
 720                 SpecimenImportConfiguratorBase config = state.getConfig();
 721                 Collection collection = null;
 722                 List<Collection> collections;
 723                 collection = (Collection) state.collections.get(collectionCode);
 724                 if (collection != null){
 725                     return collection;
 726                 }
 727                 try {
 728                     collections = getCollectionService().searchByCode(collectionCode);
 729                 } catch (Exception e) {
 730                     collections = new ArrayList<Collection>();
 731                 }
 732                 if (collections.size() > 0 && config.isReuseExistingMetaData()) {
 733                     for (Collection coll:collections){
 734                         if (coll.getCode() != null && coll.getInstitute() != null
 735                                 && coll.getCode().equalsIgnoreCase(collectionCode) && coll.getInstitute().equals(institution)) {
 736                             collection = coll;
 737                             break;
 738                         }
 739                     }
 740                 }
 741
 742                 if(collection == null){
 743                     collection =Collection.NewInstance();
 744                     collection.setCode(collectionCode);
 745                     collection.setInstitute(institution);
 746                     collection.setTitleCache(collectionCode);
 747                     UUID uuid = save(collection, state);
 748                 }
 749
 750
 751
 752                 state.collections.put(collectionCode, collection);
 753
 754                 return collection;
 755             }
 756
 757             /**
 758              * @param reference
 759              * @param citationDetail
 760              * @return
 761              */
 762             //FIXME this method is highly critical, because
 763             //  * it will have serious performance and memory problems with large databases
 764             //        (databases may easily have >1 Mio source records)
 765             //  * it does not make sense to search for existing sources and then clone them
 766             //    we need to search for existing references instead and use them (if exist)
 767             //    for our new source.
 768             protected IdentifiableSource getIdentifiableSource(Reference reference, String citationDetail) {
 769
 770               /*  List<IdentifiableSource> issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
 771
 772
 773                 if (reference != null){
 774                     try {
 775                         for (OriginalSourceBase<?> osb: issTmp){
 776                             if (osb.getCitation() != null && osb.getCitation().getTitleCache().equalsIgnoreCase(reference.getTitleCache())){
 777                                 String osbDetail = osb.getCitationMicroReference();
 778                                 if ((StringUtils.isBlank(osbDetail) && StringUtils.isBlank(citationDetail))
 779                                         || (osbDetail != null && osbDetail.equalsIgnoreCase(citationDetail)) ) {
 780 //                                  System.out.println("REFERENCE FOUND RETURN EXISTING SOURCE");
 781                                     return (IdentifiableSource) osb.clone();
 782                                 }
 783                             }
 784                         }
 785                     } catch (CloneNotSupportedException e) {
 786                         throw new RuntimeException(e);
 787                     } catch (Exception e1){
 788                         e1.printStackTrace();
 789                     }
 790                 }
 791         */
 792                 IdentifiableSource sour = IdentifiableSource.NewInstance(OriginalSourceType.Import,null,null, reference,citationDetail);
 793                 return sour;
 794             }
 795
 796             /**
 797              * Add the hierarchy for a Taxon(add higher taxa)
 798              * @param classification
 799              * @param taxon: a taxon to add as a node
 800              * @param state: the ABCD import state
 801              */
 802             protected void addParentTaxon(Taxon taxon, STATE state, boolean preferredFlag, Classification classification){
 803                 INonViralName  nvname = taxon.getName();
 804                 Rank rank = nvname.getRank();
 805                 Taxon genus =null;
 806                 Taxon subgenus =null;
 807                 Taxon species = null;
 808                 Taxon subspecies = null;
 809                 Taxon parent = null;
 810                 if(rank!=null){
 811                     if (rank.isLower(Rank.GENUS() )){
 812                         String genusOrUninomial = nvname.getGenusOrUninomial();
 813                         TaxonName taxonName = getOrCreateTaxonName(genusOrUninomial, Rank.GENUS(), preferredFlag, state, -1);
 814                         genus = getOrCreateTaxonForName(taxonName, state);
 815                         if (genus == null){
 816                             logger.debug("The genus should not be null " + taxonName);
 817                         }
 818                         if (preferredFlag) {
 819                             parent = linkParentChildNode(null, genus, classification, state);
 820                         }
 821
 822                     }
 823                     if (rank.isLower(Rank.SUBGENUS())){
 824                         String prefix = nvname.getGenusOrUninomial();
 825                         String name = nvname.getInfraGenericEpithet();
 826                         if (name != null){
 827                             TaxonName taxonName = getOrCreateTaxonName(prefix+" "+name, Rank.SUBGENUS(), preferredFlag, state, -1);
 828                             subgenus = getOrCreateTaxonForName(taxonName, state);
 829                             if (preferredFlag) {
 830                                 parent = linkParentChildNode(genus, subgenus, classification, state);
 831                             }            }
 832                     }
 833                     if (rank.isLower(Rank.SPECIES())){
 834                         if (subgenus!=null){
 835                             String prefix = nvname.getGenusOrUninomial();
 836                             String name = nvname.getInfraGenericEpithet();
 837                             String spe = nvname.getSpecificEpithet();
 838                             if (spe != null){
 839                                 TaxonName taxonName = getOrCreateTaxonName(prefix+" "+name+" "+spe, Rank.SPECIES(), preferredFlag, state, -1);
 840                                 species = getOrCreateTaxonForName(taxonName, state);
 841                                 if (preferredFlag) {
 842                                     parent = linkParentChildNode(subgenus, species, classification, state);
 843                                 }
 844                             }
 845                         }
 846                         else{
 847                             String prefix = nvname.getGenusOrUninomial();
 848                             String name = nvname.getSpecificEpithet();
 849                             if (name != null){
 850                                 TaxonName taxonName = getOrCreateTaxonName(prefix+" "+name, Rank.SPECIES(), preferredFlag, state, -1);
 851                                 species = getOrCreateTaxonForName(taxonName, state);
 852                                 if (preferredFlag) {
 853                                     parent = linkParentChildNode(genus, species, classification, state);
 854                                 }
 855                             }
 856                         }
 857                     }
 858                     if (rank.isLower(Rank.INFRASPECIES())){
 859                         TaxonName taxonName = getOrCreateTaxonName(nvname.getFullTitleCache(), Rank.SUBSPECIES(), preferredFlag, state, -1);
 860                         subspecies = getOrCreateTaxonForName(taxonName, state);
 861                         if (preferredFlag) {
 862                             parent = linkParentChildNode(species, subspecies, classification, state);
 863                         }
 864                     }
 865                 }else{
 866                     //handle cf. and aff. taxa
 867                     String genusEpithet = null;
 868                     if (nvname.getTitleCache().contains("cf.")){
 869                         genusEpithet = nvname.getTitleCache().substring(0, nvname.getTitleCache().indexOf("cf."));
 870                     } else if (nvname.getTitleCache().contains("aff.")){
 871                         genusEpithet = nvname.getTitleCache().substring(0, nvname.getTitleCache().indexOf("aff."));
 872                     }
 873                     if (genusEpithet != null){
 874                     genusEpithet = genusEpithet.trim();
 875                     TaxonName taxonName = null;
 876                     if (genusEpithet.contains(" ")){
 877                         taxonName = getOrCreateTaxonName(genusEpithet, Rank.SPECIES(), preferredFlag, state, -1);
 878                     }else{
 879                         taxonName = getOrCreateTaxonName(genusEpithet, Rank.GENUS(), preferredFlag, state, -1);
 880                     }
 881                     genus = getOrCreateTaxonForName(taxonName, state);
 882                     if (genus == null){
 883                         logger.debug("The genus should not be null " + taxonName);
 884                     }
 885                     if (preferredFlag) {
 886                         parent = linkParentChildNode(null, genus, classification, state);
 887                     }
 888                     }
 889                 }
 890                 if (preferredFlag && parent!=taxon ) {
 891                     linkParentChildNode(parent, taxon, classification, state);
 892                 }
 893             }
 894
 895             /**
 896              * Link a parent to a child and save it in the current classification
 897              * @param parent: the higher Taxon
 898              * @param child : the lower (or current) Taxon
 899              * return the Taxon from the new created Node
 900              * @param classification
 901              * @param state
 902              */
 903             protected Taxon linkParentChildNode(Taxon parent, Taxon child, Classification classification, STATE state) {
 904                 TaxonNode node =null;
 905                 if (parent != null) {
 906                     parent = (Taxon) getTaxonService().find(parent.getUuid());
 907                     child = (Taxon) getTaxonService().find(child.getUuid());
 908                     //here we do not have to check if the taxon nodes already exists
 909                     //this is done by classification.addParentChild()
 910                     //do not add child node if it already exists
 911                     if(hasTaxonNodeInClassification(child, classification)){
 912                         return child;
 913                     }
 914                     else{
 915                         node = classification.addParentChild(parent, child, state.getRef(), "");
 916                         save(node, state);
 917                     }
 918                 }
 919                 else {
 920                     if (child == null){
 921                         logger.debug("The child should not be null!");
 922                     }
 923                     child = (Taxon) getTaxonService().find(child.getUuid());
 924                     //do not add child node if it already exists
 925                     if(hasTaxonNodeInClassification(child, classification)){
 926                         return child;
 927                     }
 928                     else{
 929                         node = classification.addChildTaxon(child, state.getRef(), null);
 930                         save(node, state);
 931                     }
 932                 }
 933                 if(node!=null){
 934                     state.getReport().addTaxonNode(node);
 935                     return node.getTaxon();
 936                 }
 937                 String message = "Could not create taxon node for " +child;
 938                 state.getReport().addInfoMessage(message);
 939                 logger.warn(message);
 940                 return null;
 941             }
 942
 943             protected Taxon getOrCreateTaxonForName(TaxonName taxonName, STATE state){
 944                 if (taxonName != null){
 945                 Set<Taxon> acceptedTaxa = taxonName.getTaxa();
 946                 if(acceptedTaxa.size()>0){
 947                     Taxon firstAcceptedTaxon = acceptedTaxa.iterator().next();
 948                     if(acceptedTaxa.size()>1){
 949                         String message = "More than one accepted taxon was found for taxon name: "
 950                                 + taxonName.getTitleCache() + "!\n" + firstAcceptedTaxon + "was chosen for "+state.getDerivedUnitBase();
 951                         state.getReport().addInfoMessage(message);
 952                         logger.warn(message);
 953                     }
 954                     else{
 955                         return firstAcceptedTaxon;
 956                     }
 957                 }
 958                 else{
 959                     Set<TaxonBase> taxonAndSynonyms = taxonName.getTaxonBases();
 960                     for (TaxonBase taxonBase : taxonAndSynonyms) {
 961                         if(taxonBase.isInstanceOf(Synonym.class)){
 962                             Synonym synonym = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
 963                             Taxon acceptedTaxonOfSynonym = synonym.getAcceptedTaxon();
 964                             if(acceptedTaxonOfSynonym == null){
 965                                 String message = "No accepted taxon could be found for taxon name: "
 966                                         + taxonName.getTitleCache()
 967                                         + "!";
 968                                 state.getReport().addInfoMessage(message);
 969                                 logger.warn(message);
 970                             }
 971                             else{
 972                                 return acceptedTaxonOfSynonym;
 973                             }
 974                         }
 975                     }
 976                 }
 977                 Taxon taxon = Taxon.NewInstance(taxonName, state.getRef());
 978                 save(taxon, state);
 979                 state.getReport().addTaxon(taxon);
 980                 logger.info("Created new taxon "+ taxon);
 981                 return taxon;
 982                 }
 983                 return null;
 984
 985             }
 986
 987             private boolean hasTaxonNodeInClassification(Taxon taxon, Classification classification){
 988                 if(taxon.getTaxonNodes()!=null){
 989                     for (TaxonNode node : taxon.getTaxonNodes()){
 990                         if(node.getClassification().equals(classification)){
 991                             return true;
 992                         }
 993                     }
 994                 }
 995                 return false;
 996             }
 997
 998             /**
 999              * HandleIdentifications : get the scientific names present in the ABCD
1000              * document and store link them with the observation/specimen data
1001              * @param state: the current ABCD import state
1002              * @param derivedUnitFacade : the current derivedunitfacade
1003              */
1004             protected void handleIdentifications(STATE state, DerivedUnitFacade derivedUnitFacade) {
1005                SpecimenImportConfiguratorBase config = state.getConfig();
1006
1007
1008                 String scientificName = "";
1009                 boolean preferredFlag = false;
1010
1011                 if (state.getDataHolder().getNomenclatureCode() == ""){
1012                     if (config.getNomenclaturalCode() != null){
1013                         if (config.getNomenclaturalCode() != null){
1014                             state.getDataHolder().setNomenclatureCode(config.getNomenclaturalCode().toString());
1015
1016                         }
1017                     }
1018                 }
1019
1020                 for (int i = 0; i < state.getDataHolder().getIdentificationList().size(); i++) {
1021                     Identification identification = state.getDataHolder().getIdentificationList().get(i);
1022                     scientificName = identification.getScientificName().replaceAll(" et ", " & ");
1023
1024                     String preferred = identification.getPreferred();
1025                     preferredFlag = false;
1026                     if (preferred != null || state.getDataHolder().getIdentificationList().size()==1){
1027                         if (state.getDataHolder().getIdentificationList().size()==1){
1028                             preferredFlag = true;
1029                         }else if (preferred != null && (preferred.equals("1") || preferred.toLowerCase().indexOf("true") != -1) ) {
1030                         preferredFlag = true;
1031                     }
1032
1033                     }
1034                     if (identification.getCode() != null){
1035                     if (identification.getCode().indexOf(':') != -1) {
1036                         state.getDataHolder().setNomenclatureCode(identification.getCode().split(COLON)[1]);
1037                     }
1038                     else{
1039                         state.getDataHolder().setNomenclatureCode(identification.getCode());
1040                     }
1041                     }
1042                     TaxonName taxonName = getOrCreateTaxonName(scientificName, null, preferredFlag, state, i);
1043                     Taxon taxon = getOrCreateTaxonForName(taxonName, state);
1044                     addTaxonNode(taxon, state,preferredFlag);
1045                     linkDeterminationEvent(state, taxon, preferredFlag, derivedUnitFacade, identification.getIdentifier(), identification.getDate(), identification.getModifier());
1046                 }
1047             }
1048
1049             /**
1050              * @param taxon : a taxon to add as a node
1051              * @param state : the ABCD import state
1052              */
1053             protected void addTaxonNode(Taxon taxon, STATE state, boolean preferredFlag) {
1054                 SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
1055                 logger.info("link taxon to a taxonNode "+taxon.getTitleCache());
1056                 //only add nodes if not already existing in current classification or default classification
1057
1058                 //check if node exists in current classification
1059                 //NOTE: we cannot use hasTaxonNodeInClassification() here because we are first creating it here
1060                 if (!existsInClassification(taxon,state.getClassification(), state)){
1061                     if(config.isMoveNewTaxaToDefaultClassification()){
1062                         //check if node exists in default classification
1063                         if (!existsInClassification(taxon, state.getDefaultClassification(true), state)){
1064                             addParentTaxon(taxon, state, preferredFlag, state.getDefaultClassification(true));
1065                         }
1066                     }else{
1067                         //add non-existing taxon to current classification
1068                         addParentTaxon(taxon, state, preferredFlag, state.getClassification());
1069                     }
1070
1071                 }
1072             }
1073
1074
1075             private boolean existsInClassification(Taxon taxon, Classification classification, STATE state){
1076                 boolean exist = false;
1077                 ICdmRepository cdmAppController = state.getConfig().getCdmAppController();
1078             if(cdmAppController==null){
1079                 cdmAppController = this;
1080             }
1081             if (classification != null){
1082                 if (!taxon.getTaxonNodes().isEmpty()){
1083                     for (TaxonNode node:taxon.getTaxonNodes()){
1084                         if (node.getClassification().equals(classification)){
1085                             return true;
1086                         }
1087                     }
1088                 }
1089 // we do not need this because we already searched for taxa in db in the previous steps
1090 //              List<UuidAndTitleCache<TaxonNode>> uuidAndTitleCacheOfAllTaxa = cdmAppController.getClassificationService().getTaxonNodeUuidAndTitleCacheOfAcceptedTaxaByClassification(classification.getUuid());
1091 //              if (uuidAndTitleCacheOfAllTaxa != null){
1092 //                      for (UuidAndTitleCache p : uuidAndTitleCacheOfAllTaxa){
1093 //                          try{
1094 //                              if(p.getTitleCache().equals(taxon.getTitleCache())) {
1095 //                                  exist = true;
1096 //                              }
1097 //                          }
1098 //                          catch(Exception e){
1099 //                              logger.warn("TaxonNode doesn't seem to have a taxon");
1100 //                          }
1101 //                      }
1102 //              }
1103             }
1104                 return exist;
1105             }
1106
1107             /**
1108              * join DeterminationEvent to the Taxon Object
1109              * @param state : the ABCD import state
1110              * @param taxon: the current Taxon
1111              * @param preferredFlag :if the current name is preferred
1112              * @param derivedFacade : the derived Unit Facade
1113              */
1114             @SuppressWarnings("rawtypes")
1115         protected void linkDeterminationEvent(STATE state, Taxon taxon, boolean preferredFlag,  DerivedUnitFacade derivedFacade, String identifierStr, String dateStr, String modifier) {
1116                 SpecimenImportConfiguratorBase config = state.getConfig();
1117                 if (logger.isDebugEnabled()){
1118                     logger.info("start linkdetermination with taxon:" + taxon.getUuid()+", "+taxon);
1119                 }
1120
1121                 DeterminationEvent determinationEvent = DeterminationEvent.NewInstance();
1122                 //determinationEvent.setTaxon(taxon);
1123                 determinationEvent.setTaxonName(taxon.getName());
1124                 determinationEvent.setPreferredFlag(preferredFlag);
1125
1126
1127                 determinationEvent.setIdentifiedUnit(state.getDerivedUnitBase());
1128                 if (state.getPersonStore().get(identifierStr) != null){
1129                     determinationEvent.setActor((AgentBase)state.getPersonStore().get(identifierStr));
1130                 } else if (identifierStr != null){
1131                     Person identifier = Person.NewTitledInstance(identifierStr);
1132                     determinationEvent.setActor(identifier);
1133                 }
1134                 if (dateStr != null){
1135                     determinationEvent.setTimeperiod(TimePeriodParser.parseString(dateStr));
1136                 }
1137                 if (modifier != null){
1138                     if (modifier.equals("cf.")){
1139                         determinationEvent.setModifier(DefinedTerm.DETERMINATION_MODIFIER_CONFER());
1140                     }else if (modifier.equals("aff.")){
1141                         determinationEvent.setModifier(DefinedTerm.DETERMINATION_MODIFIER_AFFINIS());
1142                     }
1143                 }
1144                 state.getDerivedUnitBase().addDetermination(determinationEvent);
1145
1146                 if (logger.isDebugEnabled()){
1147                     logger.debug("NB TYPES INFO: "+ state.getDataHolder().getStatusList().size());
1148                 }
1149                 for (SpecimenTypeDesignationStatus specimenTypeDesignationstatus : state.getDataHolder().getStatusList()) {
1150                     if (specimenTypeDesignationstatus != null) {
1151                         if (logger.isDebugEnabled()){
1152                             logger.debug("specimenTypeDesignationstatus :"+ specimenTypeDesignationstatus);
1153                         }
1154
1155                         ICdmRepository cdmAppController = config.getCdmAppController();
1156                         if(cdmAppController == null){
1157                             cdmAppController = this;
1158                         }
1159                         specimenTypeDesignationstatus = HibernateProxyHelper.deproxy(cdmAppController.getTermService().find(specimenTypeDesignationstatus.getUuid()), SpecimenTypeDesignationStatus.class);
1160                         //Designation
1161                         TaxonName name = taxon.getName();
1162                         SpecimenTypeDesignation designation = SpecimenTypeDesignation.NewInstance();
1163
1164                         designation.setTypeStatus(specimenTypeDesignationstatus);
1165                         designation.setTypeSpecimen(state.getDerivedUnitBase());
1166                         name.addTypeDesignation(designation, true);
1167                     }
1168                 }
1169                 save(state.getDerivedUnitBase(), state);
1170
1171                 for (String[] fullReference : state.getDataHolder().getReferenceList()) {
1172
1173
1174                     String strReference=fullReference[0];
1175                     String citationDetail = fullReference[1];
1176                     String citationURL = fullReference[2];
1177                     List<Reference> references = getReferenceService().listByTitleWithRestrictions(Reference.class, "strReference", MatchMode.EXACT, null, null, null, null, null);
1178
1179                     if (!references.isEmpty()){
1180                         Reference reference = null;
1181                         for (Reference refe: references) {
1182                             if (refe.getTitleCache().equalsIgnoreCase(strReference)) {
1183                                 reference =refe;
1184                                 break;
1185                             }
1186                         }
1187                         if (reference ==null){
1188                             reference = ReferenceFactory.newGeneric();
1189                             reference.setTitleCache(strReference, true);
1190                             save(reference, state);
1191                         }
1192                         determinationEvent.addReference(reference);
1193                     }
1194                 }
1195                 save(state.getDerivedUnitBase(), state);
1196
1197                 if (config.isAddIndividualsAssociationsSuchAsSpecimenAndObservations() && preferredFlag) {
1198                     //do not add IndividualsAssociation to non-preferred taxa
1199                     if (logger.isDebugEnabled()){
1200                         logger.debug("isDoCreateIndividualsAssociations");
1201                     }
1202
1203                     makeIndividualsAssociation(state, taxon, determinationEvent);
1204
1205                     save(state.getDerivedUnitBase(), state);
1206                 }
1207             }
1208
1209             /**
1210              * create and link each association (specimen, observation..) to the accepted taxon
1211              * @param state : the ABCD import state
1212              * @param taxon: the current Taxon
1213              * @param determinationEvent:the determinationevent
1214              */
1215             protected void makeIndividualsAssociation(STATE state, Taxon taxon, DeterminationEvent determinationEvent) {
1216                 SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
1217                 SpecimenUserInteraction sui = config.getSpecimenUserInteraction();
1218
1219                 if (logger.isDebugEnabled()){
1220                     logger.info("MAKE INDIVIDUALS ASSOCIATION");
1221                 }
1222
1223                 TaxonDescription taxonDescription = null;
1224                 Set<TaxonDescription> descriptions= taxon.getDescriptions();
1225                if (!descriptions.isEmpty()){ taxonDescription = descriptions.iterator().next();}
1226
1227 //                  for (TaxonDescription description : descriptions){
1228 //                      Set<IdentifiableSource> sources =  new HashSet<>();
1229 //                      sources.addAll(description.getTaxon().getSources());
1230 //                      sources.addAll(description.getSources());
1231 //                      for (IdentifiableSource source:sources){
1232 //                          if(state.getRef().equals(source.getCitation())) {
1233 //                              taxonDescription = description;
1234 //                          }
1235 //                      }
1236 //                  }
1237               //  }
1238                 if (taxonDescription == null){
1239                     taxonDescription = TaxonDescription.NewInstance(taxon, false);
1240                     if(sourceNotLinkedToElement(taxonDescription,state.getRef(),null)) {
1241                         taxonDescription.addSource(OriginalSourceType.Import, null, null, state.getRef(), null);
1242                     }
1243                     state.setDescriptionGroup(taxonDescription);
1244                     taxon.addDescription(taxonDescription);
1245                 }
1246
1247                 //PREPARE REFERENCE QUESTIONS
1248
1249                 Map<String,OriginalSourceBase<?>> sourceMap = new HashMap<String, OriginalSourceBase<?>>();
1250
1251                 List<IdentifiableSource> issTmp = new ArrayList<>();//getCommonService().list(IdentifiableSource.class, null, null, null, null);
1252                 List<DescriptionElementSource> issTmp2 = new ArrayList<>();//getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1253
1254                 Set<OriginalSourceBase> osbSet = new HashSet<OriginalSourceBase>();
1255                 if(issTmp2!=null) {
1256                     osbSet.addAll(issTmp2);
1257                 }
1258                 if(issTmp!=null) {
1259                     osbSet.addAll(issTmp);
1260                 }
1261
1262
1263                 addToSourceMap(sourceMap, osbSet);
1264
1265 //              if (((Abcd206ImportConfigurator) state.getConfig()).isInteractWithUser()){
1266 //                  List<OriginalSourceBase<?>> res = null;
1267 //                  if(!state.isDescriptionSourcesSet()){
1268 //                      res = sui.askForSource(sourceMap, "the description group ("+taxon+")",
1269 //                              "The current reference is "+state.getRef().getTitleCache(),getReferenceService(), state.getDataHolder().docSources);
1270 //                      state.setDescriptionRefs(res);
1271 //                      state.setDescriptionSourcesSet(true);
1272 //                  }
1273 //                  else{
1274 //                      res=state.getDescriptionRefs();
1275 //                  }
1276 //                  if(res !=null) {
1277 //                      for (OriginalSourceBase<?> sour:res){
1278 //                          if(sour.isInstanceOf(IdentifiableSource.class)){
1279 //                              try {
1280 //                                  if(sourceNotLinkedToElement(taxonDescription,sour)) {
1281 //                                      taxonDescription.addSource((IdentifiableSource)sour.clone());
1282 //                                  }
1283 //                              } catch (CloneNotSupportedException e) {
1284 //                                  logger.warn("no cloning?");
1285 //                              }
1286 //                          }else{
1287 //                              if(sourceNotLinkedToElement(taxonDescription,sour)) {
1288 //                                  taxonDescription.addSource(OriginalSourceType.Import,null, null, sour.getCitation(),sour.getCitationMicroReference());
1289 //                              }
1290 //                          }
1291 //                      }
1292 //                  }
1293 //              }
1294 //              else {
1295                     if(sourceNotLinkedToElement(taxonDescription,state.getRef(),null)) {
1296                         taxonDescription.addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1297                     }
1298 //              }
1299                 state.setDescriptionGroup(taxonDescription);
1300
1301                 IndividualsAssociation indAssociation = IndividualsAssociation.NewInstance();
1302                 Feature feature = makeFeature(state.getDerivedUnitBase());
1303                 indAssociation.setAssociatedSpecimenOrObservation(state.getDerivedUnitBase());
1304                 indAssociation.setFeature(feature);
1305 //
1306 //              if (((Abcd206ImportConfigurator) state.getConfig()).isInteractWithUser()){
1307 //                  sourceMap = new HashMap<String, OriginalSourceBase<?>>();
1308 //
1309 //                  issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
1310 //                  issTmp2 = getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1311 //
1312 //                  osbSet = new HashSet<OriginalSourceBase>();
1313 //                  if(issTmp2!=null) {
1314 //                      osbSet.addAll(issTmp2);
1315 //                  }
1316 //                  if(issTmp!=null) {
1317 //                      osbSet.addAll(issTmp);
1318 //                  }
1319 //
1320 //
1321 //                  addToSourceMap(sourceMap, osbSet);
1322 //
1323 //                  List<OriginalSourceBase<?>> sources =null;
1324 //                  if(!state.isAssociationSourcesSet()) {
1325 //                      sources = sui.askForSource(sourceMap,  "descriptive element (association) ",taxon.toString(),
1326 //                              getReferenceService(),state.getDataHolder().getDocSources());
1327 //                      state.setAssociationRefs(sources);
1328 //                      state.setAssociationSourcesSet(true);
1329 //                  }
1330 //                  else{
1331 //                      sources=state.getAssociationRefs();
1332 //                  }
1333 //                  if(sources !=null) {
1334 //                      for (OriginalSourceBase<?> source: sources) {
1335 //                          if(source !=null) {
1336 //                              if(source.isInstanceOf(DescriptionElementSource.class)){
1337 //                                  try {
1338 //                                      if(sourceNotLinkedToElement(indAssociation,source)) {
1339 //                                          indAssociation.addSource((DescriptionElementSource)source.clone());
1340 //                                      }
1341 //                                  } catch (CloneNotSupportedException e) {
1342 //                                      logger.warn("clone forbidden?");
1343 //                                  }
1344 //                              }else{
1345 //                                  if(sourceNotLinkedToElement(indAssociation,source)) {
1346 //                                      indAssociation.addSource(OriginalSourceType.Import,null, null, source.getCitation(),source.getCitationMicroReference());
1347 //                                  }
1348 //                                  try {
1349 //                                      if(sourceNotLinkedToElement(state.getDerivedUnitBase(), source)) {
1350 //                                          state.getDerivedUnitBase().addSource((IdentifiableSource) source.clone());
1351 //                                      }
1352 //                                  } catch (CloneNotSupportedException e) {
1353 //                                      // TODO Auto-generated catch block
1354 //                                      e.printStackTrace();
1355 //                                  }
1356 //                              }
1357 //
1358 //                          }
1359 //                      }
1360 //                  }
1361 //              }else {
1362                     if(sourceNotLinkedToElement(indAssociation,state.getImportReference(state.getActualAccessPoint()),null)) {
1363                         indAssociation.addSource(OriginalSourceType.Import,null, null, state.getImportReference(state.getActualAccessPoint()), null);
1364                     }
1365                     if(sourceNotLinkedToElement(state.getDerivedUnitBase(), state.getImportReference(state.getActualAccessPoint()),null)) {
1366                         state.getDerivedUnitBase().addSource(OriginalSourceType.Import,null, null, state.getImportReference(state.getActualAccessPoint()), null);
1367                     }
1368                     for (Reference citation : determinationEvent.getReferences()) {
1369                         if(sourceNotLinkedToElement(indAssociation,citation,null))
1370                         {
1371                             indAssociation.addSource(DescriptionElementSource.NewInstance(OriginalSourceType.Import, null, null, citation, null));
1372                         }
1373                         if(sourceNotLinkedToElement(state.getDerivedUnitBase(), state.getImportReference(state.getActualAccessPoint()),null)) {
1374                             state.getDerivedUnitBase().addSource(OriginalSourceType.Import,null, null, state.getImportReference(state.getActualAccessPoint()), null);
1375                         }
1376                     }
1377            //     }
1378
1379                 taxonDescription.addElement(indAssociation);
1380
1381                 save(taxonDescription, state);
1382                 save(taxon, state);
1383                 state.getReport().addDerivate(state.getDerivedUnitBase(), config);
1384                 state.getReport().addIndividualAssociation(taxon, state.getDataHolder().getUnitID(), state.getDerivedUnitBase());
1385             }
1386
1387             /**
1388              * @param derivedUnitBase2
1389              * @param ref2
1390              * @param object
1391              * @return
1392              */
1393             private boolean sourceNotLinkedToElement(DerivedUnit derivedUnitBase2, Reference b, String d) {
1394                 Set<IdentifiableSource> linkedSources = derivedUnitBase2.getSources();
1395                 for (IdentifiableSource is:linkedSources){
1396                     Reference a = is.getCitation();
1397                     String c = is.getCitationMicroReference();
1398
1399                     boolean refMatch=false;
1400                     boolean microMatch=false;
1401
1402                     try{
1403                         if (a==null && b==null) {
1404                             refMatch=true;
1405                         }
1406                         if (a!=null && b!=null) {
1407                             if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1408                                 refMatch=true;
1409                             }
1410                         }
1411                     }catch(Exception e){}
1412
1413
1414                     try{
1415                         if (c==null && d==null) {
1416                             microMatch=true;
1417                         }
1418                         if(c!=null && d!=null) {
1419                             if(c.equalsIgnoreCase(d)) {
1420                                 microMatch=true;
1421                             }
1422                         }
1423                     }
1424                     catch(Exception e){}
1425
1426                     if (microMatch && refMatch) {
1427                         return false;
1428                     }
1429
1430
1431                 }
1432                 return true;
1433             }
1434
1435             private <T extends OriginalSourceBase<?>> boolean  sourceNotLinkedToElement(ISourceable<T> sourcable, Reference reference, String microReference) {
1436                 Set<T> linkedSources = sourcable.getSources();
1437                 for (T is:linkedSources){
1438                     Reference unitReference = is.getCitation();
1439                     String unitMicroReference = is.getCitationMicroReference();
1440
1441                     boolean refMatch=false;
1442                     boolean microMatch=false;
1443
1444                     try{
1445                         if (unitReference==null && reference==null) {
1446                             refMatch=true;
1447                         }
1448                         if (unitReference!=null && reference!=null) {
1449                             if (unitReference.getTitleCache().equalsIgnoreCase(reference.getTitleCache())) {
1450                                 refMatch=true;
1451                             }
1452                         }
1453                     }catch(Exception e){}
1454
1455                     try{
1456                         if (unitMicroReference==null && microReference==null) {
1457                             microMatch=true;
1458                         }
1459                         if(unitMicroReference!=null && microReference!=null) {
1460                             if(unitMicroReference.equalsIgnoreCase(microReference)) {
1461                                 microMatch=true;
1462                             }
1463                         }
1464                     }
1465                     catch(Exception e){}
1466
1467                     if (microMatch && refMatch) {
1468                         return false;
1469                     }
1470                 }
1471                 return true;
1472             }
1473
1474             /**
1475              * look for the Feature object (FieldObs, Specimen,...)
1476              * @param unit : a specimen or obersvation base
1477              * @return the corresponding Feature
1478              */
1479             private Feature makeFeature(SpecimenOrObservationBase<?> unit) {
1480                 SpecimenOrObservationType type = unit.getRecordBasis();
1481
1482
1483
1484                 if (type.isFeatureObservation()){
1485                     return Feature.OBSERVATION();
1486                 }else if (type.isFeatureSpecimen()){
1487                     return Feature.SPECIMEN();
1488                 }else if (type == SpecimenOrObservationType.DerivedUnit){
1489                     return Feature.OBSERVATION();
1490                     //            return getFeature("Specimen or observation");
1491                 }else{
1492                     String message = "Unhandled record basis '%s' for defining individuals association feature type. Use default.";
1493                     logger.warn(String.format(message, type.getMessage()));
1494                     return Feature.OBSERVATION();
1495                     //            return getFeature("Specimen or observation");
1496
1497                 }
1498             }
1499
1500
1501             /**
1502              * @param sourceMap
1503              * @param osbSet
1504              */
1505             protected void addToSourceMap(Map<String, OriginalSourceBase<?>> sourceMap, Set<OriginalSourceBase> osbSet) {
1506                 for( OriginalSourceBase<?> osb:osbSet) {
1507                     if(osb.getCitation()!=null && osb.getCitationMicroReference() !=null  && !osb.getCitationMicroReference().isEmpty()) {
1508                         try{
1509                             sourceMap.put(osb.getCitation().getTitleCache()+ "---"+osb.getCitationMicroReference(),osb);
1510                         }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1511                     } else if(osb.getCitation()!=null){
1512                         try{
1513                             sourceMap.put(osb.getCitation().getTitleCache(),osb);
1514                         }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1515                     }
1516                 }
1517             }
1518
1519
1520
1521
1522 }