cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/SpecimenImportBase.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9 package eu.etaxonomy.cdm.io.specimen;
  10
  11 import java.util.ArrayList;
  12 import java.util.EnumSet;
  13 import java.util.HashMap;
  14 import java.util.HashSet;
  15 import java.util.List;
  16 import java.util.Map;
  17 import java.util.Set;
  18 import java.util.UUID;
  19
  20 import org.apache.logging.log4j.LogManager;
  21 import org.apache.logging.log4j.Logger;
  22
  23 import eu.etaxonomy.cdm.api.application.ICdmRepository;
  24 import eu.etaxonomy.cdm.api.service.config.FindOccurrencesConfigurator;
  25 import eu.etaxonomy.cdm.api.service.pager.Pager;
  26 import eu.etaxonomy.cdm.facade.DerivedUnitFacade;
  27 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  28 import eu.etaxonomy.cdm.io.common.CdmImportBase;
  29 import eu.etaxonomy.cdm.io.common.IImportConfigurator;
  30 import eu.etaxonomy.cdm.io.specimen.abcd206.in.Identification;
  31 import eu.etaxonomy.cdm.io.specimen.abcd206.in.SpecimenImportReport;
  32 import eu.etaxonomy.cdm.model.agent.AgentBase;
  33 import eu.etaxonomy.cdm.model.agent.Institution;
  34 import eu.etaxonomy.cdm.model.agent.Person;
  35 import eu.etaxonomy.cdm.model.agent.Team;
  36 import eu.etaxonomy.cdm.model.common.CdmBase;
  37 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  38 import eu.etaxonomy.cdm.model.common.LanguageString;
  39 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  40 import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
  41 import eu.etaxonomy.cdm.model.description.DescriptionType;
  42 import eu.etaxonomy.cdm.model.description.Feature;
  43 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  44 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  45 import eu.etaxonomy.cdm.model.name.INonViralName;
  46 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  47 import eu.etaxonomy.cdm.model.name.Rank;
  48 import eu.etaxonomy.cdm.model.name.RankClass;
  49 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
  50 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
  51 import eu.etaxonomy.cdm.model.name.TaxonName;
  52 import eu.etaxonomy.cdm.model.name.TaxonNameFactory;
  53 import eu.etaxonomy.cdm.model.occurrence.Collection;
  54 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  55 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
  56 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
  57 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationType;
  58 import eu.etaxonomy.cdm.model.reference.ISourceable;
  59 import eu.etaxonomy.cdm.model.reference.OriginalSourceBase;
  60 import eu.etaxonomy.cdm.model.reference.OriginalSourceType;
  61 import eu.etaxonomy.cdm.model.reference.Reference;
  62 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  63 import eu.etaxonomy.cdm.model.taxon.Classification;
  64 import eu.etaxonomy.cdm.model.taxon.Synonym;
  65 import eu.etaxonomy.cdm.model.taxon.Taxon;
  66 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  67 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  68 import eu.etaxonomy.cdm.model.term.DefinedTerm;
  69 import eu.etaxonomy.cdm.persistence.query.MatchMode;
  70 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
  71 import eu.etaxonomy.cdm.strategy.parser.ParserProblem;
  72 import eu.etaxonomy.cdm.strategy.parser.TimePeriodParser;
  73
  74 /**
  75  * @author p.kelbert
  76  * @since 20.10.2008
  77  */
  78 public abstract class SpecimenImportBase<CONFIG extends IImportConfigurator, STATE extends SpecimenImportStateBase>
  79         extends CdmImportBase<CONFIG, STATE> {
  80
  81     private static final long serialVersionUID = 4423065367998125678L;
  82     private static final Logger logger = LogManager.getLogger();
  83
  84         protected static final UUID SPECIMEN_SCAN_TERM = UUID.fromString("acda15be-c0e2-4ea8-8783-b9b0c4ad7f03");
  85
  86         private static final String COLON = ":";
  87
  88         protected Map<String, DefinedTerm> kindOfUnitsMap;
  89
  90
  91
  92         @Override
  93     protected abstract void doInvoke(STATE state);
  94
  95         /**
  96          * Handle a single unit
  97          * @param state
  98          * @param item
  99          */
 100         protected abstract void handleSingleUnit(STATE state, Object item) ;
 101
 102
 103
 104         protected TaxonName getOrCreateTaxonName(String scientificName, Rank rank, boolean preferredFlag, STATE state, int unitIndexInAbcdFile){
 105             TaxonName taxonName = null;
 106         SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
 107
 108         //check atomised name data for rank
 109         //new name will be created
 110         TaxonName atomisedTaxonName = null;
 111         if (rank==null && unitIndexInAbcdFile>=0 && ((state.getDataHolder().getAtomisedIdentificationList() != null && !state.getDataHolder().getAtomisedIdentificationList().isEmpty())|| state.getDataHolder().getAtomisedIdentificationList().size() > 0)) {
 112             atomisedTaxonName = setTaxonNameByType(state.getDataHolder().getAtomisedIdentificationList().get(unitIndexInAbcdFile), scientificName, state);
 113             if(atomisedTaxonName!=null){
 114                 rank = atomisedTaxonName.getRank();
 115             }
 116         }
 117         if(config.isReuseExistingTaxaWhenPossible()){
 118             TaxonName parsedName = atomisedTaxonName;
 119             if(parsedName==null){
 120
 121                 parsedName = parseScientificName(scientificName, state, state.getReport(), rank);
 122
 123             }
 124             atomisedTaxonName = parsedName;
 125             if(config.isIgnoreAuthorship() && parsedName!=null){// && preferredFlag){
 126                 // do not ignore authorship for non-preferred names because they need
 127                 // to be created for the determination history
 128                 String nameCache = TaxonName.castAndDeproxy(parsedName).getNameCache();
 129                 List<TaxonName> names = getNameService().findNamesByNameCache(nameCache, MatchMode.EXACT, null);
 130                 if (!names.isEmpty()){
 131                      taxonName = getBestMatchingName(scientificName, new ArrayList<>(names), state);
 132                 }
 133                 if (taxonName == null && !names.isEmpty()){
 134                     taxonName = names.get(0);
 135                 }
 136
 137             } else {
 138                 //search for existing names
 139                 List<TaxonName> names = getNameService().listByTitleWithRestrictions(TaxonName.class, scientificName, MatchMode.EXACT, null, null, null, null, null);
 140                 taxonName = getBestMatchingName(scientificName, names, state);
 141                 //still nothing found -> try with the atomised name full title cache
 142                 if(taxonName==null && atomisedTaxonName!=null){
 143                     names = getNameService().listByTitleWithRestrictions(TaxonName.class, atomisedTaxonName.getFullTitleCache(), MatchMode.EXACT, null, null, null, null, null);
 144                     taxonName = getBestMatchingName(atomisedTaxonName.getTitleCache(), names, state);
 145                     //still nothing found -> try with the atomised name title cache
 146                     if(taxonName==null){
 147                         names = getNameService().listByTitleWithRestrictions(TaxonName.class, atomisedTaxonName.getTitleCache(), MatchMode.EXACT, null, null, null, null, null);
 148                         taxonName = getBestMatchingName(atomisedTaxonName.getTitleCache(), names, state);
 149                     }
 150                 }
 151
 152             }
 153
 154         }
 155
 156         if(taxonName==null && atomisedTaxonName!=null){
 157             taxonName = atomisedTaxonName;
 158             state.getReport().addName(taxonName);
 159             logger.info("Created new taxon name "+taxonName);
 160             if(taxonName.hasProblem()){
 161                 state.getReport().addInfoMessage(String.format("Created %s with parsing problems", taxonName));
 162             }
 163             if(!atomisedTaxonName.getTitleCache().equals(scientificName)){
 164                 state.getReport().addInfoMessage(String.format("Taxon %s was parsed as %s", scientificName, atomisedTaxonName.getTitleCache()));
 165             }
 166         }
 167         else if(taxonName==null){
 168             //create new taxon name
 169
 170             if (state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNAFP)){
 171                 taxonName = TaxonNameFactory.NewBotanicalInstance(rank);
 172             }else if (state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICZN)){
 173                 taxonName = TaxonNameFactory.NewZoologicalInstance(rank);
 174             }else{
 175                 taxonName = TaxonNameFactory.NewNonViralInstance(rank);
 176             }
 177             taxonName.setFullTitleCache(scientificName,true);
 178             taxonName.setTitleCache(scientificName, true);
 179             state.getReport().addName(taxonName);
 180             logger.info("Created new taxon name "+taxonName);
 181         }
 182         if (taxonName != null){
 183             state.names.put(taxonName.getNameCache(), taxonName);
 184         }
 185         if(!taxonName.isPersisted()) {
 186             save(taxonName, state);
 187         }
 188         return taxonName;
 189     }
 190
 191          protected TaxonName getBestMatchingName(String scientificName, java.util.Collection<TaxonName> names, STATE state){
 192                 Set<TaxonName> namesWithAcceptedTaxa = new HashSet<>();
 193                 List<TaxonName> namesWithAcceptedTaxaInClassification = new ArrayList<>();
 194                 for (TaxonName name : names) {
 195                     if(!name.getTaxa().isEmpty()){
 196                         Set<Taxon> taxa = name.getTaxa();
 197                         for (Taxon taxon:taxa){
 198                             if (!taxon.getTaxonNodes().isEmpty()){
 199                                 //use only taxa included in a classification
 200                                 for (TaxonNode node:taxon.getTaxonNodes()){
 201                                     if (state.getClassification() != null && node.getClassification().equals(state.getClassification())){
 202                                         namesWithAcceptedTaxaInClassification.add(name);
 203                                     }else {
 204                                         namesWithAcceptedTaxa.add(name);
 205                                     }
 206                                 }
 207
 208                             }
 209                         }
 210
 211                     }
 212                 }
 213                 String message = String.format("More than one taxon name was found for %s, maybe in other classifications!", scientificName);
 214                 //check for names with accepted taxa in classification
 215                 if(namesWithAcceptedTaxaInClassification.size()>0){
 216                 if(namesWithAcceptedTaxaInClassification.size()>1){
 217
 218                     state.getReport().addInfoMessage(message);
 219                     logger.warn(message);
 220                     return null;
 221                 }
 222                 return namesWithAcceptedTaxaInClassification.iterator().next();
 223             }
 224               //check for any names with accepted taxa
 225                 if(namesWithAcceptedTaxa.size()>0){
 226                     if(namesWithAcceptedTaxa.size()>1){
 227
 228                         state.getReport().addInfoMessage(message);
 229                         logger.warn(message);
 230                         return null;
 231                     }
 232                     return namesWithAcceptedTaxa.iterator().next();
 233                 }
 234                 //no names with accepted taxa found -> check accepted taxa of synonyms
 235                 List<Taxon> taxaFromSynonyms = new ArrayList<>();
 236                 for (TaxonName name : names) {
 237                     Set<TaxonBase> taxonBases = name.getTaxonBases();
 238                     for (TaxonBase taxonBase : taxonBases) {
 239                         if(taxonBase.isInstanceOf(Synonym.class)){
 240                             Synonym synonym = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
 241                             taxaFromSynonyms.add(synonym.getAcceptedTaxon());
 242                         }
 243                     }
 244                 }
 245                 if(taxaFromSynonyms.size()>0){
 246                     if(taxaFromSynonyms.size()>1){
 247                         state.getReport().addInfoMessage(message);
 248                         logger.warn(message);
 249                         return null;
 250                     }
 251                     return taxaFromSynonyms.iterator().next().getName();
 252                 }
 253                 //no accepted and no synonyms -> return one of the names and create a new taxon
 254                 if (names.isEmpty()){
 255                     return null;
 256                 }else{
 257                     return names.iterator().next();
 258                 }
 259             }
 260          /**
 261              * Parse automatically the scientific name
 262              * @param scientificName the scientific name to parse
 263              * @param state the current import state
 264              * @param report the import report
 265              * @return a parsed name
 266              */
 267
 268             protected TaxonName parseScientificName(String scientificName, STATE state, SpecimenImportReport report, Rank rank) {
 269
 270                 NonViralNameParserImpl nvnpi = NonViralNameParserImpl.NewInstance();
 271                 TaxonName taxonName = null;
 272                 boolean problem = false;
 273
 274                 if (logger.isDebugEnabled()){
 275                     logger.debug("parseScientificName " + state.getDataHolder().getNomenclatureCode().toString());
 276                 }
 277
 278                 if (state.getDataHolder().getNomenclatureCode() != null && (state.getDataHolder().getNomenclatureCode().toString().equals("Zoological") || state.getDataHolder().getNomenclatureCode().toString().contains("ICZN"))) {
 279                     taxonName = (TaxonName)nvnpi.parseFullName(scientificName, NomenclaturalCode.ICZN, rank);
 280                     if (taxonName.hasProblem()) {
 281                         problem = true;
 282                     }
 283                 }
 284                 else if (state.getDataHolder().getNomenclatureCode() != null && (state.getDataHolder().getNomenclatureCode().toString().equals("Botanical") || state.getDataHolder().getNomenclatureCode().toString().contains("ICBN")  || state.getDataHolder().getNomenclatureCode().toString().contains("ICNAFP"))) {
 285                     taxonName = (TaxonName)nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNAFP, rank);
 286                     if (taxonName.hasProblem()) {
 287                         problem = true;
 288                     }
 289                 }
 290                 else if (state.getDataHolder().getNomenclatureCode() != null && (state.getDataHolder().getNomenclatureCode().toString().equals("Bacterial") || state.getDataHolder().getNomenclatureCode().toString().contains("ICBN"))) {
 291                     taxonName = (TaxonName)nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNP, rank);
 292                     if (taxonName.hasProblem()) {
 293                         problem = true;
 294                     }
 295                 }
 296                 else if (state.getDataHolder().getNomenclatureCode() != null && (state.getDataHolder().getNomenclatureCode().toString().equals("Cultivar") || state.getDataHolder().getNomenclatureCode().toString().contains("ICNCP"))) {
 297                     taxonName = (TaxonName)nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNCP, rank);
 298                     if (taxonName.hasProblem()) {
 299                         problem = true;
 300                     }
 301                 }
 302                 if (problem) {
 303                     String message = String.format("Parsing problems for %s", scientificName);
 304                     if(taxonName!=null){
 305                         for (ParserProblem parserProblem : taxonName.getParsingProblems()) {
 306                             message += "\n\t- "+parserProblem;
 307                         }
 308                     }
 309                     report.addInfoMessage(message);
 310                     logger.info(message);
 311                 }
 312                 return taxonName;
 313
 314             }
 315
 316             /**
 317              * Create the name without automatic parsing, either because it failed, or because the user deactivated it.
 318              * The name is built upon the ABCD fields
 319              * @param atomisedMap : the ABCD atomised fields
 320              * @param fullName : the full scientific name
 321              * @param state
 322              * @return the corresponding Botanical or Zoological or... name
 323              */
 324             protected TaxonName setTaxonNameByType(
 325                     HashMap<String, String> atomisedMap, String fullName, STATE state) {
 326                 boolean problem = false;
 327                 if (logger.isDebugEnabled()){
 328                     logger.debug("settaxonnamebytype " + state.getDataHolder().getNomenclatureCode().toString());
 329                 }
 330
 331                 if (state.getDataHolder().getNomenclatureCode().equals("Zoological") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICZN.getUuid())) {
 332                     TaxonName taxonName = TaxonNameFactory.NewZoologicalInstance(null);
 333                     taxonName.setFullTitleCache(fullName, true);
 334                     taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
 335                     taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
 336                     taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap,"SpeciesEpithet")));
 337                     taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap,"SubspeciesEpithet")));
 338
 339                     if (taxonName.getGenusOrUninomial() != null){
 340                         taxonName.setRank(Rank.GENUS());
 341                     }
 342
 343                     if (taxonName.getInfraGenericEpithet() != null){
 344                         taxonName.setRank(Rank.SUBGENUS());
 345                     }
 346
 347                     if (taxonName.getSpecificEpithet() != null){
 348                         taxonName.setRank(Rank.SPECIES());
 349                     }
 350
 351                     if (taxonName.getInfraSpecificEpithet() != null){
 352                         taxonName.setRank(Rank.SUBSPECIES());
 353                     }
 354
 355                     Team team = null;
 356                     if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 357                         team = Team.NewInstance();
 358                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
 359                     }
 360                     else {
 361                         if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 362                             team = Team.NewInstance();
 363                             team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
 364                         }
 365                     }
 366                     if (team != null) {
 367                         taxonName.setBasionymAuthorship(team);
 368                     }
 369                     else {
 370                         if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 371                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
 372                         }
 373                         else if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 374                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamAndYear"));
 375                         }
 376                     }
 377                     if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
 378                         team = Team.NewInstance();
 379                         team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
 380                         taxonName.setCombinationAuthorship(team);
 381                     }
 382                     if (taxonName.hasProblem()) {
 383                         logger.info("pb ICZN");
 384                         problem = true;
 385                     }
 386                     else {
 387                         return taxonName;
 388                     }
 389                 }
 390                 else if (state.getDataHolder().getNomenclatureCode().equals("Botanical") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNAFP.getUuid())) {
 391                     TaxonName taxonName = parseScientificName(fullName, state, state.getReport(), null);
 392                     if (taxonName != null){
 393                         return taxonName;
 394                     }
 395                     else{
 396                         taxonName = TaxonNameFactory.NewBotanicalInstance(null);
 397                     }
 398                     taxonName.setFullTitleCache(fullName, true);
 399                     taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
 400                     taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "FirstEpithet")));
 401                     taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "InfraSpeEpithet")));
 402                     try {
 403                         taxonName.setRank(Rank.getRankByLatinName(getFromMap(atomisedMap, "Rank")));
 404                     } catch (Exception e) {
 405                         if (taxonName.getInfraSpecificEpithet() != null){
 406                             taxonName.setRank(Rank.SUBSPECIES());
 407                         }
 408                         else if (taxonName.getSpecificEpithet() != null){
 409                             taxonName.setRank(Rank.SPECIES());
 410                         }
 411                         else if (taxonName.getInfraGenericEpithet() != null){
 412                             taxonName.setRank(Rank.SUBGENUS());
 413                         }
 414                         else if (taxonName.getGenusOrUninomial() != null){
 415                             taxonName.setRank(Rank.GENUS());
 416                         }
 417                     }
 418                     Team team = null;
 419                     if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 420                         team = Team.NewInstance();
 421                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
 422                         taxonName.setBasionymAuthorship(team);
 423                     }
 424                     if (getFromMap(atomisedMap, "AuthorTeam") != null) {
 425                         team = Team.NewInstance();
 426                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeam"), true);
 427                         taxonName.setCombinationAuthorship(team);
 428                     }
 429                     if (team == null) {
 430                         if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 431                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
 432                         }
 433                         else if (getFromMap(atomisedMap, "AuthorTeam") != null) {
 434                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeam"));
 435                         }
 436                     }
 437                     if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
 438                         team = Team.NewInstance();
 439                         team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
 440                         taxonName.setCombinationAuthorship(team);
 441                     }
 442                     if (taxonName.hasProblem()) {
 443                         logger.info("pb ICBN");
 444                         problem = true;
 445                     }
 446                     else {
 447                         return taxonName;
 448                     }
 449                 }
 450                 else if (state.getDataHolder().getNomenclatureCode().equals("Bacterial") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNP.getUuid())) {
 451                     TaxonName taxonName = TaxonNameFactory.NewBacterialInstance(null);
 452                     taxonName.setFullTitleCache(fullName, true);
 453                     taxonName.setGenusOrUninomial(getFromMap(atomisedMap, "Genus"));
 454                     taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
 455                     taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "Species")));
 456                     taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "SubspeciesEpithet")));
 457
 458                     if (taxonName.getGenusOrUninomial() != null){
 459                         taxonName.setRank(Rank.GENUS());
 460                     }
 461                     else if (taxonName.getInfraGenericEpithet() != null){
 462                         taxonName.setRank(Rank.SUBGENUS());
 463                     }
 464                     else if (taxonName.getSpecificEpithet() != null){
 465                         taxonName.setRank(Rank.SPECIES());
 466                     }
 467                     else if (taxonName.getInfraSpecificEpithet() != null){
 468                         taxonName.setRank(Rank.SUBSPECIES());
 469                     }
 470
 471                     if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 472                         Team team = Team.NewInstance();
 473                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
 474                         taxonName.setCombinationAuthorship(team);
 475                     }
 476                     if (getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear") != null) {
 477                         Team team = Team.NewInstance();
 478                         team.setTitleCache(getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear"), true);
 479                         taxonName.setBasionymAuthorship(team);
 480                     }
 481                     if (taxonName.hasProblem()) {
 482                         logger.info("pb ICNP");
 483                         problem = true;
 484                     }
 485                     else {
 486                         return taxonName;
 487                     }
 488                 }
 489                 else if (state.getDataHolder().getNomenclatureCode().equals("Cultivar")) {
 490                     TaxonName taxonName = TaxonNameFactory.NewCultivarInstance(null);
 491
 492                     if (taxonName.hasProblem()) {
 493                         logger.info("pb ICNCP");
 494                         problem = true;
 495                     }
 496                     else {
 497                         return taxonName;
 498                     }
 499                     return taxonName;
 500                 }
 501
 502                 if (problem) {
 503                     logger.info("Problem im setTaxonNameByType ");
 504                     TaxonName taxonName = TaxonNameFactory.NewNonViralInstance(null);
 505                     taxonName.setFullTitleCache(fullName, true);
 506                     return taxonName;
 507                 }
 508                 TaxonName tn = TaxonNameFactory.NewNonViralInstance(null);
 509                 return tn;
 510             }
 511
 512             /**
 513              * Get a formated string from a hashmap
 514              * @param atomisedMap
 515              * @param key
 516              * @return
 517              */
 518             private String getFromMap(HashMap<String, String> atomisedMap, String key) {
 519                 String value = null;
 520                 if (atomisedMap.containsKey(key)) {
 521                     value = atomisedMap.get(key);
 522                 }
 523
 524                 try {
 525                     if (value != null && key.matches(".*Year.*")) {
 526                         value = value.trim();
 527                         if (value.matches("[a-z A-Z ]*[0-9]{4}$")) {
 528                             String tmp = value.split("[0-9]{4}$")[0];
 529                             int year = Integer.parseInt(value.split(tmp)[1]);
 530                             if (year >= 1752) {
 531                                 value = tmp;
 532                             }
 533                             else {
 534                                 value = null;
 535                             }
 536                         }
 537                         else {
 538                             value = null;
 539                         }
 540                     }
 541                 }
 542                 catch (Exception e) {
 543                     value = null;
 544                 }
 545                 return value;
 546             }
 547
 548             /**
 549              * Very fast and dirty implementation to allow handling of transient objects as described in
 550              * https://dev.e-taxonomy.eu/redmine/issues/3726
 551              *
 552              * Not yet complete.
 553              *
 554              * @param cdmBase
 555              * @param state
 556              */
 557             protected UUID save(CdmBase cdmBase, SpecimenImportStateBase state) {
 558                 ICdmRepository cdmRepository = state.getConfig().getCdmAppController();
 559                 if (cdmRepository == null){
 560                     cdmRepository = this;
 561                 }
 562
 563                 if (cdmBase.isInstanceOf(LanguageString.class)){
 564                     return cdmRepository.getTermService().saveLanguageData(CdmBase.deproxy(cdmBase, LanguageString.class));
 565                 }else if (cdmBase.isInstanceOf(SpecimenOrObservationBase.class)){
 566                     SpecimenOrObservationBase specimen = CdmBase.deproxy(cdmBase, SpecimenOrObservationBase.class);
 567
 568                     return cdmRepository.getOccurrenceService().saveOrUpdate(specimen);
 569                 }else if (cdmBase.isInstanceOf(Reference.class)){
 570                     return cdmRepository.getReferenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, Reference.class));
 571                 }else if (cdmBase.isInstanceOf(Classification.class)){
 572                     return cdmRepository.getClassificationService().saveOrUpdate(CdmBase.deproxy(cdmBase, Classification.class));
 573                 }else if (cdmBase.isInstanceOf(AgentBase.class)){
 574                     return cdmRepository.getAgentService().saveOrUpdate(CdmBase.deproxy(cdmBase, AgentBase.class));
 575                 }else if (cdmBase.isInstanceOf(Collection.class)){
 576                     return cdmRepository.getCollectionService().saveOrUpdate(CdmBase.deproxy(cdmBase, Collection.class));
 577                 }else if (cdmBase.isInstanceOf(DescriptionBase.class)){
 578                     DescriptionBase description = CdmBase.deproxy(cdmBase, DescriptionBase.class);
 579
 580                     return cdmRepository.getDescriptionService().saveOrUpdate(description);
 581                 }else if (cdmBase.isInstanceOf(TaxonBase.class)){
 582                     return cdmRepository.getTaxonService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonBase.class));
 583                 }else if (cdmBase.isInstanceOf(TaxonName.class)){
 584                     return cdmRepository.getNameService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonName.class));
 585                 }else if (cdmBase.isInstanceOf(TaxonNode.class)){
 586                 return cdmRepository.getTaxonNodeService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonNode.class));
 587             }else{
 588                     throw new IllegalArgumentException("Class not supported in save method: " + CdmBase.deproxy(cdmBase, CdmBase.class).getClass().getSimpleName());
 589                 }
 590
 591             }
 592
 593
 594             protected SpecimenOrObservationBase findExistingSpecimen(String unitId, SpecimenImportStateBase state){
 595                 ICdmRepository cdmAppController = state.getConfig().getCdmAppController();
 596                 if(cdmAppController==null){
 597                     cdmAppController = this;
 598                 }
 599                 FindOccurrencesConfigurator config = new FindOccurrencesConfigurator();
 600                 config.setSignificantIdentifier(unitId);
 601                 List<String> propertyPaths = new ArrayList<>();
 602                 propertyPaths.add("derivedFrom.*");
 603                 config.setPropertyPaths(propertyPaths);
 604                 commitTransaction(state.getTx());
 605                 state.setTx(startTransaction());
 606                 try{
 607                         Pager<SpecimenOrObservationBase> existingSpecimens = cdmAppController.getOccurrenceService().findByTitle(config);
 608                         if(!existingSpecimens.getRecords().isEmpty()){
 609                             if(existingSpecimens.getRecords().size()==1){
 610                                 return existingSpecimens.getRecords().iterator().next();
 611                             }
 612                         }
 613
 614                 }catch(NullPointerException e){
 615                         logger.error("searching for existing specimen creates NPE: " + config.getSignificantIdentifier());
 616                         e.printStackTrace();
 617                 }
 618
 619
 620                 return null;
 621             }
 622
 623             protected abstract void importAssociatedUnits(STATE state, Object item, DerivedUnitFacade derivedUnitFacade);
 624
 625             /**
 626              * getFacade : get the DerivedUnitFacade based on the recordBasis
 627              * @param state
 628              *
 629              * @return DerivedUnitFacade
 630              */
 631             protected DerivedUnitFacade getFacade(STATE state) {
 632                 if (logger.isDebugEnabled()){
 633                     logger.info("getFacade()");
 634                 }
 635                 SpecimenOrObservationType type = null;
 636
 637                 // create specimen
 638                 if (NB((state.getDataHolder().getRecordBasis())) != null) {
 639                     if (state.getDataHolder().getRecordBasis().toLowerCase().startsWith("s") || state.getDataHolder().getRecordBasis().toLowerCase().indexOf("specimen")>-1) {// specimen
 640                         type = SpecimenOrObservationType.PreservedSpecimen;
 641                     }
 642                     if (state.getDataHolder().getRecordBasis().toLowerCase().startsWith("o") ||state.getDataHolder().getRecordBasis().toLowerCase().indexOf("observation")>-1 ) {
 643                         type = SpecimenOrObservationType.Observation;
 644                     }
 645                     if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("fossil")>-1){
 646                         type = SpecimenOrObservationType.Fossil;
 647                     }
 648                     if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("living")>-1) {
 649                         type = SpecimenOrObservationType.LivingSpecimen;
 650                     }
 651                     if (type == null) {
 652                         logger.info("The basis of record does not seem to be known: " + state.getDataHolder().getRecordBasis());
 653                         type = SpecimenOrObservationType.DerivedUnit;
 654                     }
 655                     // TODO fossils?
 656                 } else {
 657                     logger.info("The basis of record is null");
 658                     type = SpecimenOrObservationType.DerivedUnit;
 659                 }
 660                 DerivedUnitFacade derivedUnitFacade = DerivedUnitFacade.NewInstance(type);
 661                 return derivedUnitFacade;
 662             }
 663
 664             /**
 665              * Look if the Institution does already exist
 666              * @param institutionCode: a string with the institutioncode
 667              * @param config : the configurator
 668              * @return the Institution (existing or new)
 669              */
 670             protected Institution getInstitution(String institutionCode, STATE state) {
 671                 SpecimenImportConfiguratorBase config = state.getConfig();
 672                 Institution institution=null;
 673                 institution = (Institution)state.institutions.get(institutionCode);
 674                 if (institution != null){
 675                     return institution;
 676                 }
 677                 List<Institution> institutions;
 678                 try {
 679                     institutions = getAgentService().searchInstitutionByCode(institutionCode);
 680
 681                 } catch (Exception e) {
 682                     institutions = new ArrayList<Institution>();
 683                     logger.warn(e);
 684                 }
 685                 if (institutions.size() > 0 && config.isReuseExistingMetaData()) {
 686                     for (Institution institut:institutions){
 687                         try{
 688                             if (institut.getCode().equalsIgnoreCase(institutionCode)) {
 689                                 institution=institut;
 690                                 break;
 691                             }
 692                         }catch(Exception e){logger.warn("no institution code in the db");}
 693                     }
 694                 }
 695                 if (logger.isDebugEnabled()){
 696                     if(institution !=null) {
 697                         logger.info("getinstitution " + institution.toString());
 698                     }
 699                 }
 700                 if (institution == null){
 701                     // create institution
 702                     institution = Institution.NewInstance();
 703                     institution.setCode(institutionCode);
 704                     institution.setTitleCache(institutionCode, true);
 705                     UUID uuid = save(institution, state);
 706                 }
 707
 708
 709                 state.institutions.put(institutionCode, institution);
 710                 return institution;
 711             }
 712
 713             /**
 714              * Look if the Collection does already exist
 715              * @param collectionCode
 716              * @param collectionCode: a string
 717              * @param config : the configurator
 718              * @return the Collection (existing or new)
 719              */
 720             protected Collection getCollection(Institution institution, String collectionCode, STATE state) {
 721                 SpecimenImportConfiguratorBase config = state.getConfig();
 722                 Collection collection = null;
 723                 List<Collection> collections;
 724                 collection = (Collection) state.collections.get(collectionCode);
 725                 if (collection != null){
 726                     return collection;
 727                 }
 728                 try {
 729                     collections = getCollectionService().searchByCode(collectionCode);
 730                 } catch (Exception e) {
 731                     collections = new ArrayList<Collection>();
 732                 }
 733                 if (collections.size() > 0 && config.isReuseExistingMetaData()) {
 734                     for (Collection coll:collections){
 735                         if (coll.getCode() != null && coll.getInstitute() != null
 736                                 && coll.getCode().equalsIgnoreCase(collectionCode) && coll.getInstitute().equals(institution)) {
 737                             collection = coll;
 738                             break;
 739                         }
 740                     }
 741                 }
 742
 743                 if(collection == null){
 744                     collection =Collection.NewInstance();
 745                     collection.setCode(collectionCode);
 746                     collection.setInstitute(institution);
 747                     collection.setTitleCache(collectionCode);
 748                     UUID uuid = save(collection, state);
 749                 }
 750
 751
 752
 753                 state.collections.put(collectionCode, collection);
 754
 755                 return collection;
 756             }
 757
 758             /**
 759              * @param reference
 760              * @param citationDetail
 761              * @return
 762              */
 763             //FIXME this method is highly critical, because
 764             //  * it will have serious performance and memory problems with large databases
 765             //        (databases may easily have >1 Mio source records)
 766             //  * it does not make sense to search for existing sources and then clone them
 767             //    we need to search for existing references instead and use them (if exist)
 768             //    for our new source.
 769             protected IdentifiableSource getIdentifiableSource(Reference reference, String citationDetail) {
 770
 771                 IdentifiableSource sour = IdentifiableSource.NewInstance(OriginalSourceType.Import,null,null, reference,citationDetail);
 772                 return sour;
 773             }
 774
 775             /**
 776              * Add the hierarchy for a Taxon(add higher taxa)
 777              * @param classification
 778              * @param taxon: a taxon to add as a node
 779              * @param state: the ABCD import state
 780              */
 781             protected void addParentTaxon(Taxon taxon, STATE state, boolean preferredFlag, Classification classification){
 782                 INonViralName  nvname = taxon.getName();
 783                 Rank rank = nvname.getRank();
 784                 Taxon genus =null;
 785                 Taxon subgenus =null;
 786                 Taxon species = null;
 787                 Taxon subspecies = null;
 788                 Taxon parent = null;
 789                 if(rank!=null){
 790                     if (rank.isLowerThan(RankClass.Genus)){
 791                         String genusOrUninomial = nvname.getGenusOrUninomial();
 792                         TaxonName taxonName = getOrCreateTaxonName(genusOrUninomial, Rank.GENUS(), preferredFlag, state, -1);
 793                         genus = getOrCreateTaxonForName(taxonName, state);
 794                         if (genus == null){
 795                             logger.debug("The genus should not be null " + taxonName);
 796                         }
 797                         if (preferredFlag) {
 798                             parent = linkParentChildNode(null, genus, classification, state);
 799                         }
 800
 801                     }
 802                     if (rank.isLower(Rank.SUBGENUS())){
 803                         String prefix = nvname.getGenusOrUninomial();
 804                         String name = nvname.getInfraGenericEpithet();
 805                         if (name != null){
 806                             TaxonName taxonName = getOrCreateTaxonName(prefix+" "+name, Rank.SUBGENUS(), preferredFlag, state, -1);
 807                             subgenus = getOrCreateTaxonForName(taxonName, state);
 808                             if (preferredFlag) {
 809                                 parent = linkParentChildNode(genus, subgenus, classification, state);
 810                             }            }
 811                     }
 812                     if (rank.isLowerThan(RankClass.Species)){
 813                         if (subgenus!=null){
 814                             String prefix = nvname.getGenusOrUninomial();
 815                             String name = nvname.getInfraGenericEpithet();
 816                             String spe = nvname.getSpecificEpithet();
 817                             if (spe != null){
 818                                 TaxonName taxonName = getOrCreateTaxonName(prefix+" "+name+" "+spe, Rank.SPECIES(), preferredFlag, state, -1);
 819                                 species = getOrCreateTaxonForName(taxonName, state);
 820                                 if (preferredFlag) {
 821                                     parent = linkParentChildNode(subgenus, species, classification, state);
 822                                 }
 823                             }
 824                         }
 825                         else{
 826                             String prefix = nvname.getGenusOrUninomial();
 827                             String name = nvname.getSpecificEpithet();
 828                             if (name != null){
 829                                 TaxonName taxonName = getOrCreateTaxonName(prefix+" "+name, Rank.SPECIES(), preferredFlag, state, -1);
 830                                 species = getOrCreateTaxonForName(taxonName, state);
 831                                 if (preferredFlag) {
 832                                     parent = linkParentChildNode(genus, species, classification, state);
 833                                 }
 834                             }
 835                         }
 836                     }
 837                     if (rank.isLower(Rank.INFRASPECIES())){
 838                         TaxonName taxonName = getOrCreateTaxonName(nvname.getFullTitleCache(), Rank.SUBSPECIES(), preferredFlag, state, -1);
 839                         subspecies = getOrCreateTaxonForName(taxonName, state);
 840                         if (preferredFlag) {
 841                             parent = linkParentChildNode(species, subspecies, classification, state);
 842                         }
 843                     }
 844                 }else{
 845                     //handle cf. and aff. taxa
 846                     String genusEpithet = null;
 847                     if (nvname.getTitleCache().contains("cf.")){
 848                         genusEpithet = nvname.getTitleCache().substring(0, nvname.getTitleCache().indexOf("cf."));
 849                     } else if (nvname.getTitleCache().contains("aff.")){
 850                         genusEpithet = nvname.getTitleCache().substring(0, nvname.getTitleCache().indexOf("aff."));
 851                     }
 852                     if (genusEpithet != null){
 853                     genusEpithet = genusEpithet.trim();
 854                     TaxonName taxonName = null;
 855                     if (genusEpithet.contains(" ")){
 856                         taxonName = getOrCreateTaxonName(genusEpithet, Rank.SPECIES(), preferredFlag, state, -1);
 857                     }else{
 858                         taxonName = getOrCreateTaxonName(genusEpithet, Rank.GENUS(), preferredFlag, state, -1);
 859                     }
 860                     genus = getOrCreateTaxonForName(taxonName, state);
 861                     if (genus == null){
 862                         logger.debug("The genus should not be null " + taxonName);
 863                     }
 864                     if (preferredFlag) {
 865                         parent = linkParentChildNode(null, genus, classification, state);
 866                     }
 867                     }
 868                 }
 869                 if (preferredFlag && parent!=taxon ) {
 870                     linkParentChildNode(parent, taxon, classification, state);
 871                 }
 872             }
 873
 874             /**
 875              * Link a parent to a child and save it in the current classification
 876              * @param parent: the higher Taxon
 877              * @param child : the lower (or current) Taxon
 878              * return the Taxon from the new created Node
 879              * @param classification
 880              * @param state
 881              */
 882             protected Taxon linkParentChildNode(Taxon parent, Taxon child, Classification classification, STATE state) {
 883                 TaxonNode node =null;
 884                 List<String> propertyPaths = new ArrayList<>();
 885             propertyPaths.add("childNodes");
 886                 if (parent != null) {
 887
 888                     parent = (Taxon) getTaxonService().load(parent.getUuid(), propertyPaths);
 889                     child = (Taxon) getTaxonService().load(child.getUuid(), propertyPaths);
 890                     //here we do not have to check if the taxon nodes already exists
 891                     //this is done by classification.addParentChild()
 892                     //do not add child node if it already exists
 893                     if(hasTaxonNodeInClassification(child, classification)){
 894                         return child;
 895                     }
 896                     else{
 897                         node = classification.addParentChild(parent, child, state.getRef(), "");
 898                         save(node, state);
 899                     }
 900                 }
 901                 else {
 902                     if (child == null){
 903                         logger.debug("The child should not be null!");
 904                     }
 905                     child = (Taxon) getTaxonService().find(child.getUuid());
 906                     //do not add child node if it already exists
 907                     if(hasTaxonNodeInClassification(child, classification)){
 908                         return child;
 909                     }
 910                     else{
 911                         node = classification.addChildTaxon(child, state.getRef(), null);
 912                         save(node, state);
 913                     }
 914                 }
 915                 if(node!=null){
 916                     state.getReport().addTaxonNode(node);
 917                     return node.getTaxon();
 918                 }
 919                 String message = "Could not create taxon node for " +child;
 920                 state.getReport().addInfoMessage(message);
 921                 logger.warn(message);
 922                 return null;
 923             }
 924
 925             protected Taxon getOrCreateTaxonForName(TaxonName taxonName, STATE state){
 926                 if (taxonName != null){
 927                 Set<Taxon> acceptedTaxa = taxonName.getTaxa();
 928                 if(acceptedTaxa.size()>0){
 929                     Taxon firstAcceptedTaxon = acceptedTaxa.iterator().next();
 930                     if(acceptedTaxa.size()>1){
 931                         String message = "More than one accepted taxon was found for taxon name: "
 932                                 + taxonName.getTitleCache() + "!\n" + firstAcceptedTaxon + "was chosen for "+state.getDerivedUnitBase();
 933                         state.getReport().addInfoMessage(message);
 934                         logger.warn(message);
 935                     }
 936                     else{
 937                         return firstAcceptedTaxon;
 938                     }
 939                 }
 940                 else{
 941                     Set<TaxonBase> taxonAndSynonyms = taxonName.getTaxonBases();
 942                     for (TaxonBase taxonBase : taxonAndSynonyms) {
 943                         if(taxonBase.isInstanceOf(Synonym.class)){
 944                             Synonym synonym = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
 945                             Taxon acceptedTaxonOfSynonym = synonym.getAcceptedTaxon();
 946                             if(acceptedTaxonOfSynonym == null){
 947                                 String message = "No accepted taxon could be found for taxon name: "
 948                                         + taxonName.getTitleCache()
 949                                         + "!";
 950                                 state.getReport().addInfoMessage(message);
 951                                 logger.warn(message);
 952                             }
 953                             else{
 954                                 return acceptedTaxonOfSynonym;
 955                             }
 956                         }
 957                     }
 958                 }
 959                 Taxon taxon = Taxon.NewInstance(taxonName, state.getRef());
 960                 save(taxon, state);
 961                 state.getReport().addTaxon(taxon);
 962                 logger.info("Created new taxon "+ taxon);
 963                 return taxon;
 964                 }
 965                 return null;
 966
 967             }
 968
 969             private boolean hasTaxonNodeInClassification(Taxon taxon, Classification classification){
 970                 if(taxon.getTaxonNodes()!=null){
 971                     for (TaxonNode node : taxon.getTaxonNodes()){
 972                         if(node.getClassification().equals(classification)){
 973                             return true;
 974                         }
 975                     }
 976                 }
 977                 return false;
 978             }
 979
 980             /**
 981              * HandleIdentifications : get the scientific names present in the ABCD
 982              * document and store link them with the observation/specimen data
 983              * @param state: the current ABCD import state
 984              * @param derivedUnitFacade : the current derivedunitfacade
 985              */
 986             protected void handleIdentifications(STATE state, DerivedUnitFacade derivedUnitFacade) {
 987
 988                 SpecimenImportConfiguratorBase config = state.getConfig();
 989
 990                 String scientificName = "";
 991                 boolean preferredFlag = false;
 992
 993                 if (state.getDataHolder().getNomenclatureCode() == ""){
 994                     if (config.getNomenclaturalCode() != null){
 995                         if (config.getNomenclaturalCode() != null){
 996                             state.getDataHolder().setNomenclatureCode(config.getNomenclaturalCode().toString());
 997
 998                         }
 999                     }
1000                 }
1001
1002                 for (int i = 0; i < state.getDataHolder().getIdentificationList().size(); i++) {
1003                     Identification identification = state.getDataHolder().getIdentificationList().get(i);
1004                     scientificName = identification.getScientificName().replaceAll(" et ", " & ");
1005
1006                     String preferred = identification.getPreferred();
1007                     preferredFlag = false;
1008                     if (preferred != null || state.getDataHolder().getIdentificationList().size()==1){
1009                         if (state.getDataHolder().getIdentificationList().size()==1){
1010                             preferredFlag = true;
1011                         }else if (preferred != null && (preferred.equals("1") || preferred.toLowerCase().indexOf("true") != -1) ) {
1012                         preferredFlag = true;
1013                     }
1014
1015                     }
1016                     if (identification.getCode() != null){
1017                     if (identification.getCode().indexOf(':') != -1) {
1018                         state.getDataHolder().setNomenclatureCode(identification.getCode().split(COLON)[1]);
1019                     }
1020                     else{
1021                         state.getDataHolder().setNomenclatureCode(identification.getCode());
1022                     }
1023                     }
1024                     TaxonName taxonName = getOrCreateTaxonName(scientificName, null, preferredFlag, state, i);
1025                     Taxon taxon = getOrCreateTaxonForName(taxonName, state);
1026                     addTaxonNode(taxon, state,preferredFlag);
1027                     linkDeterminationEvent(state, taxon, preferredFlag, derivedUnitFacade, identification.getIdentifier(), identification.getDate(), identification.getModifier());
1028                 }
1029             }
1030
1031             /**
1032              * @param taxon : a taxon to add as a node
1033              * @param state : the ABCD import state
1034              */
1035             protected void addTaxonNode(Taxon taxon, STATE state, boolean preferredFlag) {
1036                 SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
1037                 logger.info("link taxon to a taxonNode "+taxon.getTitleCache());
1038                 //only add nodes if not already existing in current classification or default classification
1039
1040                 //check if node exists in current classification
1041                 //NOTE: we cannot use hasTaxonNodeInClassification() here because we are first creating it here
1042                 if (!existsInClassification(taxon,state.getClassification(), state)){
1043                     if(config.isMoveNewTaxaToDefaultClassification()){
1044                         //check if node exists in default classification
1045                         if (!existsInClassification(taxon, state.getDefaultClassification(true), state)){
1046                             addParentTaxon(taxon, state, preferredFlag, state.getDefaultClassification(true));
1047                         }
1048                     }else{
1049                         //add non-existing taxon to current classification
1050                         addParentTaxon(taxon, state, preferredFlag, state.getClassification());
1051                     }
1052
1053                 }
1054             }
1055
1056
1057             private boolean existsInClassification(Taxon taxon, Classification classification, STATE state){
1058                 boolean exist = false;
1059                 ICdmRepository cdmAppController = state.getConfig().getCdmAppController();
1060             if(cdmAppController==null){
1061                 cdmAppController = this;
1062             }
1063             if (classification != null){
1064                 if (!taxon.getTaxonNodes().isEmpty()){
1065                     for (TaxonNode node:taxon.getTaxonNodes()){
1066                         if (node.getClassification().equals(classification)){
1067                             return true;
1068                         }
1069                     }
1070                 }
1071 // we do not need this because we already searched for taxa in db in the previous steps
1072 //              List<UuidAndTitleCache<TaxonNode>> uuidAndTitleCacheOfAllTaxa = cdmAppController.getClassificationService().getTaxonNodeUuidAndTitleCacheOfAcceptedTaxaByClassification(classification.getUuid());
1073 //              if (uuidAndTitleCacheOfAllTaxa != null){
1074 //                      for (UuidAndTitleCache p : uuidAndTitleCacheOfAllTaxa){
1075 //                          try{
1076 //                              if(p.getTitleCache().equals(taxon.getTitleCache())) {
1077 //                                  exist = true;
1078 //                              }
1079 //                          }
1080 //                          catch(Exception e){
1081 //                              logger.warn("TaxonNode doesn't seem to have a taxon");
1082 //                          }
1083 //                      }
1084 //              }
1085             }
1086                 return exist;
1087             }
1088
1089             /**
1090              * join DeterminationEvent to the Taxon Object
1091              * @param state : the ABCD import state
1092              * @param taxon: the current Taxon
1093              * @param preferredFlag :if the current name is preferred
1094              * @param derivedFacade : the derived Unit Facade
1095              */
1096             @SuppressWarnings("rawtypes")
1097         protected void linkDeterminationEvent(STATE state, Taxon taxon, boolean preferredFlag,  DerivedUnitFacade derivedFacade, String identifierStr, String dateStr, String modifier) {
1098                 SpecimenImportConfiguratorBase config = state.getConfig();
1099                 if (logger.isDebugEnabled()){
1100                     logger.info("start linkdetermination with taxon:" + taxon.getUuid()+", "+taxon);
1101                 }
1102
1103                 DeterminationEvent determinationEvent = DeterminationEvent.NewInstance();
1104                 //determinationEvent.setTaxon(taxon);
1105                 determinationEvent.setTaxonName(taxon.getName());
1106                 determinationEvent.setPreferredFlag(preferredFlag);
1107
1108
1109                 determinationEvent.setIdentifiedUnit(state.getDerivedUnitBase());
1110                 if (state.getPersonStore().get(identifierStr) != null){
1111                     determinationEvent.setActor((AgentBase)state.getPersonStore().get(identifierStr));
1112                 } else if (identifierStr != null){
1113                     Person identifier = Person.NewTitledInstance(identifierStr);
1114                     determinationEvent.setActor(identifier);
1115                 }
1116                 if (dateStr != null){
1117                     determinationEvent.setTimeperiod(TimePeriodParser.parseString(dateStr));
1118                 }
1119                 if (modifier != null){
1120                     if (modifier.equals("cf.")){
1121                         determinationEvent.setModifier(DefinedTerm.DETERMINATION_MODIFIER_CONFER());
1122                     }else if (modifier.equals("aff.")){
1123                         determinationEvent.setModifier(DefinedTerm.DETERMINATION_MODIFIER_AFFINIS());
1124                     }
1125                 }
1126                 state.getDerivedUnitBase().addDetermination(determinationEvent);
1127
1128                 if (logger.isDebugEnabled()){
1129                     logger.debug("NB TYPES INFO: "+ state.getDataHolder().getStatusList().size());
1130                 }
1131                 for (SpecimenTypeDesignationStatus specimenTypeDesignationstatus : state.getDataHolder().getStatusList()) {
1132                     if (specimenTypeDesignationstatus != null) {
1133                         if (logger.isDebugEnabled()){
1134                             logger.debug("specimenTypeDesignationstatus :"+ specimenTypeDesignationstatus);
1135                         }
1136
1137                         ICdmRepository cdmAppController = config.getCdmAppController();
1138                         if(cdmAppController == null){
1139                             cdmAppController = this;
1140                         }
1141                         specimenTypeDesignationstatus = HibernateProxyHelper.deproxy(cdmAppController.getTermService().find(specimenTypeDesignationstatus.getUuid()), SpecimenTypeDesignationStatus.class);
1142                         //Designation
1143                         TaxonName name = taxon.getName();
1144                         SpecimenTypeDesignation designation = SpecimenTypeDesignation.NewInstance();
1145
1146                         designation.setTypeStatus(specimenTypeDesignationstatus);
1147                         designation.setTypeSpecimen(state.getDerivedUnitBase());
1148                         name.addTypeDesignation(designation, false);
1149                     }
1150                 }
1151                 save(state.getDerivedUnitBase(), state);
1152
1153                 for (String[] fullReference : state.getDataHolder().getReferenceList()) {
1154
1155
1156                     String strReference=fullReference[0];
1157                     String citationDetail = fullReference[1];
1158                     String citationURL = fullReference[2];
1159                     List<Reference> references = getReferenceService().listByTitleWithRestrictions(Reference.class, "strReference", MatchMode.EXACT, null, null, null, null, null);
1160
1161                     if (!references.isEmpty()){
1162                         Reference reference = null;
1163                         for (Reference refe: references) {
1164                             if (refe.getTitleCache().equalsIgnoreCase(strReference)) {
1165                                 reference =refe;
1166                                 break;
1167                             }
1168                         }
1169                         if (reference ==null){
1170                             reference = ReferenceFactory.newGeneric();
1171                             reference.setTitleCache(strReference, true);
1172                             save(reference, state);
1173                         }
1174                         determinationEvent.addReference(reference);
1175                     }
1176                 }
1177                 save(state.getDerivedUnitBase(), state);
1178
1179                 if (config.isAddIndividualsAssociationsSuchAsSpecimenAndObservations() && preferredFlag) {
1180                     //do not add IndividualsAssociation to non-preferred taxa
1181                     if (logger.isDebugEnabled()){
1182                         logger.debug("isDoCreateIndividualsAssociations");
1183                     }
1184
1185                     makeIndividualsAssociation(state, taxon, determinationEvent);
1186
1187                     save(state.getDerivedUnitBase(), state);
1188                 }
1189             }
1190
1191             /**
1192              * create and link each association (specimen, observation..) to the accepted taxon
1193              * @param state : the ABCD import state
1194              * @param taxon: the current Taxon
1195              * @param determinationEvent:the determinationevent
1196              */
1197             protected void makeIndividualsAssociation(STATE state, Taxon taxon, DeterminationEvent determinationEvent) {
1198                 SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
1199                 SpecimenUserInteraction sui = config.getSpecimenUserInteraction();
1200
1201                 if (logger.isDebugEnabled()){
1202                     logger.info("MAKE INDIVIDUALS ASSOCIATION");
1203                 }
1204
1205                 TaxonDescription taxonDescription = null;
1206                 Set<TaxonDescription> descriptions= taxon.getDescriptions();
1207                 if (state.getIndividualsAssociationDescriptionPerTaxon(taxon.getUuid()) != null){
1208                     taxonDescription = state.getIndividualsAssociationDescriptionPerTaxon(taxon.getUuid());
1209                 }
1210                if (taxonDescription == null && !descriptions.isEmpty() && state.getConfig().isReuseExistingDescriptiveGroups()){
1211                    for (TaxonDescription desc: descriptions){
1212                        if (desc.getTypes().contains(DescriptionType.INDIVIDUALS_ASSOCIATION)){
1213                            taxonDescription = desc;
1214                            break;
1215                        }
1216                    }
1217                }
1218
1219                if (taxonDescription == null){
1220                     taxonDescription = TaxonDescription.NewInstance(taxon, false);
1221                     taxonDescription.setTypes(EnumSet.of(DescriptionType.INDIVIDUALS_ASSOCIATION));
1222                     if(sourceNotLinkedToElement(taxonDescription,state.getRef(),null)) {
1223                         taxonDescription.addSource(OriginalSourceType.Import, null, null, state.getRef(), null);
1224                     }
1225                     state.setIndividualsAssociationDescriptionPerTaxon(taxonDescription);
1226                     taxon.addDescription(taxonDescription);
1227                 }
1228
1229                 //PREPARE REFERENCE QUESTIONS
1230
1231                 Map<String,OriginalSourceBase> sourceMap = new HashMap<>();
1232
1233                 List<IdentifiableSource> issTmp = new ArrayList<>();//getCommonService().list(IdentifiableSource.class, null, null, null, null);
1234                 List<DescriptionElementSource> issTmp2 = new ArrayList<>();//getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1235
1236                 Set<OriginalSourceBase> osbSet = new HashSet<>();
1237                 if(issTmp2!=null) {
1238                     osbSet.addAll(issTmp2);
1239                 }
1240                 if(issTmp!=null) {
1241                     osbSet.addAll(issTmp);
1242                 }
1243
1244
1245                 addToSourceMap(sourceMap, osbSet);
1246
1247
1248             if(sourceNotLinkedToElement(taxonDescription,state.getRef(),null)) {
1249                 taxonDescription.addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1250             }
1251
1252                 state.setIndividualsAssociationDescriptionPerTaxon(taxonDescription);
1253
1254                 IndividualsAssociation indAssociation = IndividualsAssociation.NewInstance();
1255                 Feature feature = makeFeature(state.getDerivedUnitBase());
1256                 indAssociation.setAssociatedSpecimenOrObservation(state.getDerivedUnitBase());
1257                 indAssociation.setFeature(feature);
1258
1259             if(sourceNotLinkedToElement(indAssociation,state.getImportReference(state.getActualAccessPoint()),null)) {
1260                 indAssociation.addSource(OriginalSourceType.Import,null, null, state.getImportReference(state.getActualAccessPoint()), null);
1261             }
1262             if(sourceNotLinkedToElement(state.getDerivedUnitBase(), state.getImportReference(state.getActualAccessPoint()),null)) {
1263                 state.getDerivedUnitBase().addSource(OriginalSourceType.Import,null, null, state.getImportReference(state.getActualAccessPoint()), null);
1264             }
1265             for (Reference citation : determinationEvent.getReferences()) {
1266                 if(sourceNotLinkedToElement(indAssociation,citation,null))
1267                 {
1268                     indAssociation.addSource(DescriptionElementSource.NewInstance(OriginalSourceType.Import, null, null, citation, null));
1269                 }
1270                 if(sourceNotLinkedToElement(state.getDerivedUnitBase(), state.getImportReference(state.getActualAccessPoint()),null)) {
1271                     state.getDerivedUnitBase().addSource(OriginalSourceType.Import,null, null, state.getImportReference(state.getActualAccessPoint()), null);
1272                 }
1273             }
1274
1275
1276                 taxonDescription.addElement(indAssociation);
1277
1278                 save(taxonDescription, state);
1279                 save(taxon, state);
1280                 state.getReport().addDerivate(state.getDerivedUnitBase(), config);
1281                 state.getReport().addIndividualAssociation(taxon, state.getDataHolder().getUnitID(), state.getDerivedUnitBase());
1282             }
1283
1284             /**
1285              * @param derivedUnitBase2
1286              * @param ref2
1287              * @param object
1288              * @return
1289              */
1290             private boolean sourceNotLinkedToElement(DerivedUnit derivedUnitBase2, Reference b, String d) {
1291                 Set<IdentifiableSource> linkedSources = derivedUnitBase2.getSources();
1292                 for (IdentifiableSource is:linkedSources){
1293                     Reference a = is.getCitation();
1294                     String c = is.getCitationMicroReference();
1295
1296                     boolean refMatch=false;
1297                     boolean microMatch=false;
1298
1299                     try{
1300                         if (a==null && b==null) {
1301                             refMatch=true;
1302                         }
1303                         if (a!=null && b!=null) {
1304                             if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1305                                 refMatch=true;
1306                             }
1307                         }
1308                     }catch(Exception e){}
1309
1310
1311                     try{
1312                         if (c==null && d==null) {
1313                             microMatch=true;
1314                         }
1315                         if(c!=null && d!=null) {
1316                             if(c.equalsIgnoreCase(d)) {
1317                                 microMatch=true;
1318                             }
1319                         }
1320                     }
1321                     catch(Exception e){}
1322
1323                     if (microMatch && refMatch) {
1324                         return false;
1325                     }
1326
1327
1328                 }
1329                 return true;
1330             }
1331
1332             private <T extends OriginalSourceBase> boolean  sourceNotLinkedToElement(ISourceable<T> sourcable, Reference reference, String microReference) {
1333                 Set<T> linkedSources = sourcable.getSources();
1334                 for (T is:linkedSources){
1335                     Reference unitReference = is.getCitation();
1336                     String unitMicroReference = is.getCitationMicroReference();
1337
1338                     boolean refMatch=false;
1339                     boolean microMatch=false;
1340
1341                     try{
1342                         if (unitReference==null && reference==null) {
1343                             refMatch=true;
1344                         }
1345                         if (unitReference!=null && reference!=null) {
1346                             if (unitReference.getTitleCache().equalsIgnoreCase(reference.getTitleCache())) {
1347                                 refMatch=true;
1348                             }
1349                         }
1350                     }catch(Exception e){}
1351
1352                     try{
1353                         if (unitMicroReference==null && microReference==null) {
1354                             microMatch=true;
1355                         }
1356                         if(unitMicroReference!=null && microReference!=null) {
1357                             if(unitMicroReference.equalsIgnoreCase(microReference)) {
1358                                 microMatch=true;
1359                             }
1360                         }
1361                     }
1362                     catch(Exception e){}
1363
1364                     if (microMatch && refMatch) {
1365                         return false;
1366                     }
1367                 }
1368                 return true;
1369             }
1370
1371             /**
1372              * look for the Feature object (FieldObs, Specimen,...)
1373              * @param unit : a specimen or obersvation base
1374              * @return the corresponding Feature
1375              */
1376             private Feature makeFeature(SpecimenOrObservationBase<?> unit) {
1377                 SpecimenOrObservationType type = unit.getRecordBasis();
1378
1379
1380
1381                 if (type.isFeatureObservation()){
1382                     return Feature.OBSERVATION();
1383                 }else if (type.isFeatureSpecimen()){
1384                     return Feature.SPECIMEN();
1385                 }else if (type == SpecimenOrObservationType.DerivedUnit){
1386                     return Feature.OBSERVATION();
1387                     //            return getFeature("Specimen or observation");
1388                 }else{
1389                     String message = "Unhandled record basis '%s' for defining individuals association feature type. Use default.";
1390                     logger.warn(String.format(message, type.getLabel()));
1391                     return Feature.OBSERVATION();
1392                     //            return getFeature("Specimen or observation");
1393
1394                 }
1395             }
1396
1397
1398             /**
1399              * @param sourceMap
1400              * @param osbSet
1401              */
1402             protected void addToSourceMap(Map<String, OriginalSourceBase> sourceMap, Set<OriginalSourceBase> osbSet) {
1403                 for( OriginalSourceBase osb:osbSet) {
1404                     if(osb.getCitation()!=null && osb.getCitationMicroReference() !=null  && !osb.getCitationMicroReference().isEmpty()) {
1405                         try{
1406                             sourceMap.put(osb.getCitation().getTitleCache()+ "---"+osb.getCitationMicroReference(),osb);
1407                         }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1408                     } else if(osb.getCitation()!=null){
1409                         try{
1410                             sourceMap.put(osb.getCitation().getTitleCache(),osb);
1411                         }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1412                     }
1413                 }
1414             }
1415
1416
1417
1418
1419 }