cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/SpecimenImportBase.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9 package eu.etaxonomy.cdm.io.specimen;
  10
  11 import java.util.ArrayList;
  12 import java.util.EnumSet;
  13 import java.util.HashMap;
  14 import java.util.HashSet;
  15 import java.util.List;
  16 import java.util.Map;
  17 import java.util.Set;
  18 import java.util.UUID;
  19
  20 import org.apache.logging.log4j.LogManager;
  21 import org.apache.logging.log4j.Logger;
  22
  23 import eu.etaxonomy.cdm.api.application.ICdmRepository;
  24 import eu.etaxonomy.cdm.api.service.config.FindOccurrencesConfigurator;
  25 import eu.etaxonomy.cdm.api.service.pager.Pager;
  26 import eu.etaxonomy.cdm.facade.DerivedUnitFacade;
  27 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  28 import eu.etaxonomy.cdm.io.common.CdmImportBase;
  29 import eu.etaxonomy.cdm.io.common.IImportConfigurator;
  30 import eu.etaxonomy.cdm.io.specimen.abcd206.in.Identification;
  31 import eu.etaxonomy.cdm.io.specimen.abcd206.in.SpecimenImportReport;
  32 import eu.etaxonomy.cdm.model.agent.AgentBase;
  33 import eu.etaxonomy.cdm.model.agent.Institution;
  34 import eu.etaxonomy.cdm.model.agent.Person;
  35 import eu.etaxonomy.cdm.model.agent.Team;
  36 import eu.etaxonomy.cdm.model.common.CdmBase;
  37 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  38 import eu.etaxonomy.cdm.model.common.LanguageString;
  39 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  40 import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
  41 import eu.etaxonomy.cdm.model.description.DescriptionType;
  42 import eu.etaxonomy.cdm.model.description.Feature;
  43 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  44 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  45 import eu.etaxonomy.cdm.model.name.INonViralName;
  46 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  47 import eu.etaxonomy.cdm.model.name.Rank;
  48 import eu.etaxonomy.cdm.model.name.RankClass;
  49 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
  50 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
  51 import eu.etaxonomy.cdm.model.name.TaxonName;
  52 import eu.etaxonomy.cdm.model.name.TaxonNameFactory;
  53 import eu.etaxonomy.cdm.model.occurrence.Collection;
  54 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  55 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
  56 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
  57 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationType;
  58 import eu.etaxonomy.cdm.model.reference.ISourceable;
  59 import eu.etaxonomy.cdm.model.reference.OriginalSourceBase;
  60 import eu.etaxonomy.cdm.model.reference.OriginalSourceType;
  61 import eu.etaxonomy.cdm.model.reference.Reference;
  62 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  63 import eu.etaxonomy.cdm.model.taxon.Classification;
  64 import eu.etaxonomy.cdm.model.taxon.Synonym;
  65 import eu.etaxonomy.cdm.model.taxon.Taxon;
  66 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  67 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  68 import eu.etaxonomy.cdm.model.term.DefinedTerm;
  69 import eu.etaxonomy.cdm.persistence.query.MatchMode;
  70 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
  71 import eu.etaxonomy.cdm.strategy.parser.ParserProblem;
  72 import eu.etaxonomy.cdm.strategy.parser.TimePeriodParser;
  73
  74 /**
  75  * @author p.kelbert
  76  * @since 20.10.2008
  77  */
  78 public abstract class SpecimenImportBase<CONFIG extends IImportConfigurator, STATE extends SpecimenImportStateBase>
  79         extends CdmImportBase<CONFIG, STATE> {
  80
  81     private static final long serialVersionUID = 4423065367998125678L;
  82     private static final Logger logger = LogManager.getLogger();
  83
  84         protected static final UUID SPECIMEN_SCAN_TERM = UUID.fromString("acda15be-c0e2-4ea8-8783-b9b0c4ad7f03");
  85
  86         private static final String COLON = ":";
  87
  88         protected Map<String, DefinedTerm> kindOfUnitsMap;
  89
  90
  91         @Override
  92     protected abstract void doInvoke(STATE state);
  93
  94         /**
  95          * Handle a single unit
  96          */
  97         protected abstract void handleSingleUnit(STATE state, Object item) ;
  98
  99         protected TaxonName getOrCreateTaxonName(String scientificName, Rank rank, boolean preferredFlag, STATE state, int unitIndexInAbcdFile){
 100             TaxonName taxonName = null;
 101         SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
 102
 103         //check atomised name data for rank
 104         //new name will be created
 105         TaxonName atomisedTaxonName = null;
 106         if (rank==null && unitIndexInAbcdFile>=0 && ((state.getDataHolder().getAtomisedIdentificationList() != null && !state.getDataHolder().getAtomisedIdentificationList().isEmpty())|| state.getDataHolder().getAtomisedIdentificationList().size() > 0)) {
 107             atomisedTaxonName = setTaxonNameByType(state.getDataHolder().getAtomisedIdentificationList().get(unitIndexInAbcdFile), scientificName, state);
 108             if(atomisedTaxonName!=null){
 109                 rank = atomisedTaxonName.getRank();
 110             }
 111         }
 112         if(config.isReuseExistingTaxaWhenPossible()){
 113             TaxonName parsedName = atomisedTaxonName;
 114             if(parsedName==null){
 115
 116                 parsedName = parseScientificName(scientificName, state, state.getReport(), rank);
 117
 118             }
 119             atomisedTaxonName = parsedName;
 120             if(config.isIgnoreAuthorship() && parsedName!=null){// && preferredFlag){
 121                 // do not ignore authorship for non-preferred names because they need
 122                 // to be created for the determination history
 123                 String nameCache = TaxonName.castAndDeproxy(parsedName).getNameCache();
 124                 List<TaxonName> names = getNameService().findNamesByNameCache(nameCache, MatchMode.EXACT, null);
 125                 if (!names.isEmpty()){
 126                      taxonName = getBestMatchingName(scientificName, new ArrayList<>(names), state);
 127                 }
 128                 if (taxonName == null && !names.isEmpty()){
 129                     taxonName = names.get(0);
 130                 }
 131
 132             } else {
 133                 //search for existing names
 134                 List<TaxonName> names = getNameService().listByTitleWithRestrictions(TaxonName.class, scientificName, MatchMode.EXACT, null, null, null, null, null);
 135                 taxonName = getBestMatchingName(scientificName, names, state);
 136                 //still nothing found -> try with the atomised name full title cache
 137                 if(taxonName==null && atomisedTaxonName!=null){
 138                     names = getNameService().listByTitleWithRestrictions(TaxonName.class, atomisedTaxonName.getFullTitleCache(), MatchMode.EXACT, null, null, null, null, null);
 139                     taxonName = getBestMatchingName(atomisedTaxonName.getTitleCache(), names, state);
 140                     //still nothing found -> try with the atomised name title cache
 141                     if(taxonName==null){
 142                         names = getNameService().listByTitleWithRestrictions(TaxonName.class, atomisedTaxonName.getTitleCache(), MatchMode.EXACT, null, null, null, null, null);
 143                         taxonName = getBestMatchingName(atomisedTaxonName.getTitleCache(), names, state);
 144                     }
 145                 }
 146
 147             }
 148
 149         }
 150
 151         if(taxonName==null && atomisedTaxonName!=null){
 152             taxonName = atomisedTaxonName;
 153             state.getReport().addName(taxonName);
 154             logger.info("Created new taxon name "+taxonName);
 155             if(taxonName.hasProblem()){
 156                 state.getReport().addInfoMessage(String.format("Created %s with parsing problems", taxonName));
 157             }
 158             if(!atomisedTaxonName.getTitleCache().equals(scientificName)){
 159                 state.getReport().addInfoMessage(String.format("Taxon %s was parsed as %s", scientificName, atomisedTaxonName.getTitleCache()));
 160             }
 161         }
 162         else if(taxonName==null){
 163             //create new taxon name
 164
 165             if (state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNAFP)){
 166                 taxonName = TaxonNameFactory.NewBotanicalInstance(rank);
 167             }else if (state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICZN)){
 168                 taxonName = TaxonNameFactory.NewZoologicalInstance(rank);
 169             }else{
 170                 taxonName = TaxonNameFactory.NewNonViralInstance(rank);
 171             }
 172             taxonName.setFullTitleCache(scientificName,true);
 173             taxonName.setTitleCache(scientificName, true);
 174             state.getReport().addName(taxonName);
 175             logger.info("Created new taxon name "+taxonName);
 176         }
 177         if (taxonName != null){
 178             state.names.put(taxonName.getNameCache(), taxonName);
 179         }
 180         if(!taxonName.isPersisted()) {
 181             save(taxonName, state);
 182         }
 183         return taxonName;
 184     }
 185
 186         protected TaxonName getBestMatchingName(String scientificName, java.util.Collection<TaxonName> names, STATE state){
 187         Set<TaxonName> namesWithAcceptedTaxa = new HashSet<>();
 188         List<TaxonName> namesWithAcceptedTaxaInClassification = new ArrayList<>();
 189         for (TaxonName name : names) {
 190             if(!name.getTaxonBases().isEmpty()){
 191                 Set<TaxonBase> taxa = name.getTaxonBases();
 192                 for (TaxonBase taxonBase:taxa){
 193                     Taxon acceptedTaxon= null;
 194                     if (taxonBase instanceof Synonym) {
 195                         Synonym syn = (Synonym) taxonBase;
 196                         acceptedTaxon = syn.getAcceptedTaxon();
 197                     }else {
 198                         acceptedTaxon = (Taxon)taxonBase;
 199                     }
 200                     if (!(acceptedTaxon).getTaxonNodes().isEmpty()){
 201                         //use only taxa included in a classification
 202                         for (TaxonNode node:(acceptedTaxon).getTaxonNodes()){
 203                             if (state.getClassification() != null && node.getClassification().equals(state.getClassification())){
 204                                 namesWithAcceptedTaxaInClassification.add(name);
 205                             }else {
 206                                 namesWithAcceptedTaxa.add(name);
 207                             }
 208                         }
 209                     }
 210                 }
 211             }
 212         }
 213         String message = String.format("More than one taxon name was found for %s, maybe in other classifications!", scientificName);
 214         //check for names with accepted taxa in classification
 215         if(namesWithAcceptedTaxaInClassification.size()>0){
 216             if(namesWithAcceptedTaxaInClassification.size()>1){
 217
 218                 state.getReport().addInfoMessage(message);
 219                 logger.warn(message);
 220                 return null;
 221             }
 222             return namesWithAcceptedTaxaInClassification.iterator().next();
 223         }
 224         //check for any names with accepted taxa
 225         if(namesWithAcceptedTaxa.size()>0){
 226             if(namesWithAcceptedTaxa.size()>1){
 227
 228                 state.getReport().addInfoMessage(message);
 229                 logger.warn(message);
 230                 return null;
 231             }
 232             return namesWithAcceptedTaxa.iterator().next();
 233         }
 234 //              //no names with accepted taxa found -> check accepted taxa of synonyms -> this is handled in the first block now!
 235 //              List<Taxon> taxaFromSynonyms = new ArrayList<>();
 236 //              for (TaxonName name : names) {
 237 //                  Set<TaxonBase> taxonBases = name.getTaxonBases();
 238 //                  for (TaxonBase taxonBase : taxonBases) {
 239 //                      if(taxonBase.isInstanceOf(Synonym.class)){
 240 //                          Synonym synonym = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
 241 //                          taxaFromSynonyms.add(synonym.getAcceptedTaxon());
 242 //                      }
 243 //                  }
 244 //              }
 245 //              if(taxaFromSynonyms.size()>0){
 246 //                  if(taxaFromSynonyms.size()>1){
 247 //                      state.getReport().addInfoMessage(message);
 248 //                      logger.warn(message);
 249 //                      return null;
 250 //                  }
 251 //                  return taxaFromSynonyms.iterator().next().getName();
 252 //              }
 253                 //no accepted and no synonyms -> return one of the names and create a new taxon
 254         if (names.isEmpty()){
 255             return null;
 256         }else{
 257             return names.iterator().next();
 258         }
 259     }
 260
 261         /**
 262      * Parse automatically the scientific name
 263      * @param scientificName the scientific name to parse
 264      * @param state the current import state
 265      * @param report the import report
 266      * @return a parsed name
 267      */
 268     protected TaxonName parseScientificName(String scientificName, STATE state, SpecimenImportReport report, Rank rank) {
 269
 270         NonViralNameParserImpl nvnpi = NonViralNameParserImpl.NewInstance();
 271         TaxonName taxonName = null;
 272         boolean problem = false;
 273
 274         if (logger.isDebugEnabled()){
 275             logger.debug("parseScientificName " + state.getDataHolder().getNomenclatureCode().toString());
 276         }
 277
 278         if (state.getDataHolder().getNomenclatureCode() != null && (state.getDataHolder().getNomenclatureCode().toString().equals("Zoological") || state.getDataHolder().getNomenclatureCode().toString().contains("ICZN"))) {
 279             taxonName = (TaxonName)nvnpi.parseFullName(scientificName, NomenclaturalCode.ICZN, rank);
 280             if (taxonName.hasProblem()) {
 281                 problem = true;
 282             }
 283         }
 284         else if (state.getDataHolder().getNomenclatureCode() != null && (state.getDataHolder().getNomenclatureCode().toString().equals("Botanical") || state.getDataHolder().getNomenclatureCode().toString().contains("ICBN")  || state.getDataHolder().getNomenclatureCode().toString().contains("ICNAFP"))) {
 285             taxonName = (TaxonName)nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNAFP, rank);
 286             if (taxonName.hasProblem()) {
 287                 problem = true;
 288             }
 289         }
 290         else if (state.getDataHolder().getNomenclatureCode() != null && (state.getDataHolder().getNomenclatureCode().toString().equals("Bacterial") || state.getDataHolder().getNomenclatureCode().toString().contains("ICBN"))) {
 291             taxonName = (TaxonName)nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNP, rank);
 292             if (taxonName.hasProblem()) {
 293                 problem = true;
 294             }
 295         }
 296         else if (state.getDataHolder().getNomenclatureCode() != null && (state.getDataHolder().getNomenclatureCode().toString().equals("Cultivar") || state.getDataHolder().getNomenclatureCode().toString().contains("ICNCP"))) {
 297             taxonName = (TaxonName)nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNCP, rank);
 298             if (taxonName.hasProblem()) {
 299                 problem = true;
 300             }
 301         }
 302         if (problem) {
 303             String message = String.format("Parsing problems for %s", scientificName);
 304             if(taxonName!=null){
 305                 for (ParserProblem parserProblem : taxonName.getParsingProblems()) {
 306                     message += "\n\t- "+parserProblem;
 307                 }
 308             }
 309             report.addInfoMessage(message);
 310             logger.info(message);
 311         }
 312         return taxonName;
 313
 314     }
 315
 316     /**
 317      * Create the name without automatic parsing, either because it failed, or because the user deactivated it.
 318      * The name is built upon the ABCD fields
 319      * @param atomisedMap : the ABCD atomised fields
 320      * @param fullName : the full scientific name
 321      * @param state
 322      * @return the corresponding Botanical or Zoological or... name
 323      */
 324     protected TaxonName setTaxonNameByType(
 325             HashMap<String, String> atomisedMap, String fullName, STATE state) {
 326         boolean problem = false;
 327         if (logger.isDebugEnabled()){
 328             logger.debug("settaxonnamebytype " + state.getDataHolder().getNomenclatureCode().toString());
 329         }
 330
 331         if (state.getDataHolder().getNomenclatureCode().equals("Zoological") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICZN.getUuid())) {
 332             TaxonName taxonName = TaxonNameFactory.NewZoologicalInstance(null);
 333             taxonName.setFullTitleCache(fullName, true);
 334             taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
 335             taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
 336             taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap,"SpeciesEpithet")));
 337             taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap,"SubspeciesEpithet")));
 338
 339             if (taxonName.getGenusOrUninomial() != null){
 340                 taxonName.setRank(Rank.GENUS());
 341             }
 342
 343             if (taxonName.getInfraGenericEpithet() != null){
 344                 taxonName.setRank(Rank.SUBGENUS());
 345             }
 346
 347             if (taxonName.getSpecificEpithet() != null){
 348                 taxonName.setRank(Rank.SPECIES());
 349             }
 350
 351             if (taxonName.getInfraSpecificEpithet() != null){
 352                 taxonName.setRank(Rank.SUBSPECIES());
 353             }
 354
 355             Team team = null;
 356             if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 357                 team = Team.NewInstance();
 358                 team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
 359             }
 360             else {
 361                 if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 362                     team = Team.NewInstance();
 363                     team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
 364                 }
 365             }
 366             if (team != null) {
 367                 taxonName.setBasionymAuthorship(team);
 368             }
 369             else {
 370                 if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 371                     taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
 372                 }
 373                 else if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 374                     taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamAndYear"));
 375                 }
 376             }
 377             if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
 378                 team = Team.NewInstance();
 379                 team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
 380                 taxonName.setCombinationAuthorship(team);
 381             }
 382             if (taxonName.hasProblem()) {
 383                 logger.info("pb ICZN");
 384                 problem = true;
 385             }
 386             else {
 387                 return taxonName;
 388             }
 389         }
 390         else if (state.getDataHolder().getNomenclatureCode().equals("Botanical") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNAFP.getUuid())) {
 391             TaxonName taxonName = parseScientificName(fullName, state, state.getReport(), null);
 392             if (taxonName != null){
 393                 return taxonName;
 394             }
 395             else{
 396                 taxonName = TaxonNameFactory.NewBotanicalInstance(null);
 397             }
 398             taxonName.setFullTitleCache(fullName, true);
 399             taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
 400             taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "FirstEpithet")));
 401             taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "InfraSpeEpithet")));
 402             try {
 403                 taxonName.setRank(Rank.getRankByLatinName(getFromMap(atomisedMap, "Rank")));
 404             } catch (Exception e) {
 405                 if (taxonName.getInfraSpecificEpithet() != null){
 406                     taxonName.setRank(Rank.SUBSPECIES());
 407                 }
 408                 else if (taxonName.getSpecificEpithet() != null){
 409                     taxonName.setRank(Rank.SPECIES());
 410                 }
 411                 else if (taxonName.getInfraGenericEpithet() != null){
 412                     taxonName.setRank(Rank.SUBGENUS());
 413                 }
 414                 else if (taxonName.getGenusOrUninomial() != null){
 415                     taxonName.setRank(Rank.GENUS());
 416                 }
 417             }
 418             Team team = null;
 419             if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 420                 team = Team.NewInstance();
 421                 team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
 422                 taxonName.setBasionymAuthorship(team);
 423             }
 424             if (getFromMap(atomisedMap, "AuthorTeam") != null) {
 425                 team = Team.NewInstance();
 426                 team.setTitleCache(getFromMap(atomisedMap, "AuthorTeam"), true);
 427                 taxonName.setCombinationAuthorship(team);
 428             }
 429             if (team == null) {
 430                 if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 431                     taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
 432                 }
 433                 else if (getFromMap(atomisedMap, "AuthorTeam") != null) {
 434                     taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeam"));
 435                 }
 436             }
 437             if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
 438                 team = Team.NewInstance();
 439                 team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
 440                 taxonName.setCombinationAuthorship(team);
 441             }
 442             if (taxonName.hasProblem()) {
 443                 logger.info("pb ICBN");
 444                 problem = true;
 445             }
 446             else {
 447                 return taxonName;
 448             }
 449         }
 450         else if (state.getDataHolder().getNomenclatureCode().equals("Bacterial") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNP.getUuid())) {
 451             TaxonName taxonName = TaxonNameFactory.NewBacterialInstance(null);
 452             taxonName.setFullTitleCache(fullName, true);
 453             taxonName.setGenusOrUninomial(getFromMap(atomisedMap, "Genus"));
 454             taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
 455             taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "Species")));
 456             taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "SubspeciesEpithet")));
 457
 458             if (taxonName.getGenusOrUninomial() != null){
 459                 taxonName.setRank(Rank.GENUS());
 460             }
 461             else if (taxonName.getInfraGenericEpithet() != null){
 462                 taxonName.setRank(Rank.SUBGENUS());
 463             }
 464             else if (taxonName.getSpecificEpithet() != null){
 465                 taxonName.setRank(Rank.SPECIES());
 466             }
 467             else if (taxonName.getInfraSpecificEpithet() != null){
 468                 taxonName.setRank(Rank.SUBSPECIES());
 469             }
 470
 471             if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 472                 Team team = Team.NewInstance();
 473                 team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
 474                 taxonName.setCombinationAuthorship(team);
 475             }
 476             if (getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear") != null) {
 477                 Team team = Team.NewInstance();
 478                 team.setTitleCache(getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear"), true);
 479                 taxonName.setBasionymAuthorship(team);
 480             }
 481             if (taxonName.hasProblem()) {
 482                 logger.info("pb ICNP");
 483                 problem = true;
 484             }
 485             else {
 486                 return taxonName;
 487             }
 488         }
 489         else if (state.getDataHolder().getNomenclatureCode().equals("Cultivar")) {
 490             TaxonName taxonName = TaxonNameFactory.NewCultivarInstance(null);
 491
 492             if (taxonName.hasProblem()) {
 493                 logger.info("pb ICNCP");
 494                 problem = true;
 495             }
 496             else {
 497                 return taxonName;
 498             }
 499             return taxonName;
 500         }
 501
 502         if (problem) {
 503             logger.info("Problem im setTaxonNameByType ");
 504             TaxonName taxonName = TaxonNameFactory.NewNonViralInstance(null);
 505             taxonName.setFullTitleCache(fullName, true);
 506             return taxonName;
 507         }
 508         TaxonName tn = TaxonNameFactory.NewNonViralInstance(null);
 509         return tn;
 510     }
 511
 512     /**
 513      * Get a formated string from a hashmap
 514      * @param atomisedMap
 515      * @param key
 516      * @return
 517      */
 518     private String getFromMap(HashMap<String, String> atomisedMap, String key) {
 519         String value = null;
 520         if (atomisedMap.containsKey(key)) {
 521             value = atomisedMap.get(key);
 522         }
 523
 524         try {
 525             if (value != null && key.matches(".*Year.*")) {
 526                 value = value.trim();
 527                 if (value.matches("[a-z A-Z ]*[0-9]{4}$")) {
 528                     String tmp = value.split("[0-9]{4}$")[0];
 529                     int year = Integer.parseInt(value.split(tmp)[1]);
 530                     if (year >= 1752) {
 531                         value = tmp;
 532                     }
 533                     else {
 534                         value = null;
 535                     }
 536                 }
 537                 else {
 538                     value = null;
 539                 }
 540             }
 541         }
 542         catch (Exception e) {
 543             value = null;
 544         }
 545         return value;
 546     }
 547
 548     /**
 549      * Very fast and dirty implementation to allow handling of transient objects as described in
 550      * https://dev.e-taxonomy.eu/redmine/issues/3726
 551      *
 552      * Not yet complete.
 553      */
 554     protected UUID save(CdmBase cdmBase, SpecimenImportStateBase<?,?> state) {
 555         ICdmRepository cdmRepository = state.getConfig().getCdmAppController();
 556         if (cdmRepository == null){
 557             cdmRepository = this;
 558         }
 559
 560         if (cdmBase.isInstanceOf(LanguageString.class)){
 561             return cdmRepository.getTermService().saveLanguageData(CdmBase.deproxy(cdmBase, LanguageString.class));
 562         }else if (cdmBase.isInstanceOf(SpecimenOrObservationBase.class)){
 563             SpecimenOrObservationBase<?> specimen = CdmBase.deproxy(cdmBase, SpecimenOrObservationBase.class);
 564             return cdmRepository.getOccurrenceService().saveOrUpdate(specimen);
 565         }else if (cdmBase.isInstanceOf(Reference.class)){
 566             return cdmRepository.getReferenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, Reference.class));
 567         }else if (cdmBase.isInstanceOf(Classification.class)){
 568             return cdmRepository.getClassificationService().saveOrUpdate(CdmBase.deproxy(cdmBase, Classification.class));
 569         }else if (cdmBase.isInstanceOf(AgentBase.class)){
 570             return cdmRepository.getAgentService().saveOrUpdate(CdmBase.deproxy(cdmBase, AgentBase.class));
 571         }else if (cdmBase.isInstanceOf(Collection.class)){
 572             return cdmRepository.getCollectionService().saveOrUpdate(CdmBase.deproxy(cdmBase, Collection.class));
 573         }else if (cdmBase.isInstanceOf(DescriptionBase.class)){
 574             DescriptionBase<?> description = CdmBase.deproxy(cdmBase, DescriptionBase.class);
 575             return cdmRepository.getDescriptionService().saveOrUpdate(description);
 576         }else if (cdmBase.isInstanceOf(TaxonBase.class)){
 577             return cdmRepository.getTaxonService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonBase.class));
 578         }else if (cdmBase.isInstanceOf(TaxonName.class)){
 579             return cdmRepository.getNameService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonName.class));
 580         }else if (cdmBase.isInstanceOf(TaxonNode.class)){
 581             return cdmRepository.getTaxonNodeService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonNode.class));
 582         }else{
 583             throw new IllegalArgumentException("Class not supported in save method: " + CdmBase.deproxy(cdmBase, CdmBase.class).getClass().getSimpleName());
 584         }
 585     }
 586
 587     protected SpecimenOrObservationBase<?> findExistingSpecimen(String unitId, SpecimenImportStateBase<?,?> state){
 588         ICdmRepository cdmAppController = state.getConfig().getCdmAppController();
 589         if(cdmAppController==null){
 590             cdmAppController = this;
 591         }
 592         FindOccurrencesConfigurator config = new FindOccurrencesConfigurator();
 593         config.setSignificantIdentifier(unitId);
 594         List<String> propertyPaths = new ArrayList<>();
 595         propertyPaths.add("derivedFrom.*");
 596         config.setPropertyPaths(propertyPaths);
 597         commitTransaction(state.getTx());
 598         state.setTx(startTransaction());
 599         try{
 600                 @SuppressWarnings("rawtypes")
 601             Pager<SpecimenOrObservationBase> existingSpecimens = cdmAppController.getOccurrenceService().findByTitle(config);
 602                 if(!existingSpecimens.getRecords().isEmpty()){
 603                     if(existingSpecimens.getRecords().size()==1){
 604                         return existingSpecimens.getRecords().iterator().next();
 605                     }
 606                 }
 607
 608         }catch(NullPointerException e){
 609                 logger.error("searching for existing specimen creates NPE: " + config.getSignificantIdentifier());
 610                 e.printStackTrace();
 611         }
 612
 613
 614         return null;
 615     }
 616
 617     protected abstract void importAssociatedUnits(STATE state, Object item, DerivedUnitFacade derivedUnitFacade);
 618
 619     /**
 620      * getFacade : get the DerivedUnitFacade based on the recordBasis
 621      * @param state
 622      *
 623      * @return DerivedUnitFacade
 624      */
 625     protected DerivedUnitFacade getFacade(STATE state) {
 626         if (logger.isDebugEnabled()){
 627             logger.info("getFacade()");
 628         }
 629         SpecimenOrObservationType type = null;
 630
 631         // create specimen
 632         if (NB((state.getDataHolder().getRecordBasis())) != null) {
 633             if (state.getDataHolder().getRecordBasis().toLowerCase().startsWith("s") || state.getDataHolder().getRecordBasis().toLowerCase().indexOf("specimen")>-1) {// specimen
 634                 type = SpecimenOrObservationType.PreservedSpecimen;
 635             }
 636             if (state.getDataHolder().getRecordBasis().toLowerCase().startsWith("o") ||state.getDataHolder().getRecordBasis().toLowerCase().indexOf("observation")>-1 ) {
 637                 type = SpecimenOrObservationType.Observation;
 638             }
 639             if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("fossil")>-1){
 640                 type = SpecimenOrObservationType.Fossil;
 641             }
 642             if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("living")>-1) {
 643                 type = SpecimenOrObservationType.LivingSpecimen;
 644             }
 645             if (type == null) {
 646                 logger.info("The basis of record does not seem to be known: " + state.getDataHolder().getRecordBasis());
 647                 type = SpecimenOrObservationType.DerivedUnit;
 648             }
 649             // TODO fossils?
 650         } else {
 651             logger.info("The basis of record is null");
 652             type = SpecimenOrObservationType.DerivedUnit;
 653         }
 654         DerivedUnitFacade derivedUnitFacade = DerivedUnitFacade.NewInstance(type);
 655         return derivedUnitFacade;
 656     }
 657
 658     /**
 659      * Look if the Institution does already exist
 660      * @param institutionCode: a string with the institutioncode
 661      * @param config : the configurator
 662      * @return the Institution (existing or new)
 663      */
 664     protected Institution getInstitution(String institutionCode, STATE state) {
 665         SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
 666         Institution institution=null;
 667         institution = (Institution)state.institutions.get(institutionCode);
 668         if (institution != null){
 669             return institution;
 670         }
 671         List<Institution> institutions;
 672         try {
 673             institutions = getAgentService().searchInstitutionByCode(institutionCode);
 674
 675         } catch (Exception e) {
 676             institutions = new ArrayList<Institution>();
 677             logger.warn(e);
 678         }
 679         if (institutions.size() > 0 && config.isReuseExistingMetaData()) {
 680             for (Institution institut:institutions){
 681                 try{
 682                     if (institut.getCode().equalsIgnoreCase(institutionCode)) {
 683                         institution=institut;
 684                         break;
 685                     }
 686                 }catch(Exception e){logger.warn("no institution code in the db");}
 687             }
 688         }
 689         if (logger.isDebugEnabled()){
 690             if(institution !=null) {
 691                 logger.info("getinstitution " + institution.toString());
 692             }
 693         }
 694         if (institution == null){
 695             // create institution
 696             institution = Institution.NewInstance();
 697             institution.setCode(institutionCode);
 698             institution.setTitleCache(institutionCode, true);
 699             save(institution, state);
 700         }
 701
 702         state.institutions.put(institutionCode, institution);
 703         return institution;
 704     }
 705
 706     /**
 707      * Look if the Collection does already exist
 708      * @param collectionCode
 709      * @param collectionCode: a string
 710      * @param config : the configurator
 711      * @return the Collection (existing or new)
 712      */
 713     protected Collection getCollection(Institution institution, String collectionCode, STATE state) {
 714
 715         SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
 716         Collection collection = null;
 717         List<Collection> collections;
 718         collection = (Collection) state.collections.get(collectionCode);
 719         if (collection != null){
 720             return collection;
 721         }
 722         try {
 723             collections = getCollectionService().searchByCode(collectionCode);
 724         } catch (Exception e) {
 725             collections = new ArrayList<>();
 726         }
 727         if (collections.size() > 0 && config.isReuseExistingMetaData()) {
 728             for (Collection coll:collections){
 729                 if (coll.getCode() != null && coll.getInstitute() != null
 730                         && coll.getCode().equalsIgnoreCase(collectionCode) && coll.getInstitute().equals(institution)) {
 731                     collection = coll;
 732                     break;
 733                 }
 734             }
 735         }
 736
 737         if(collection == null){
 738             collection =Collection.NewInstance();
 739             collection.setCode(collectionCode);
 740             collection.setInstitute(institution);
 741             save(collection, state);
 742         }
 743
 744         state.collections.put(collectionCode, collection);
 745
 746         return collection;
 747     }
 748
 749     /**
 750      * @param reference
 751      * @param citationDetail
 752      * @return
 753      */
 754     //FIXME this method is highly critical, because
 755     //  * it will have serious performance and memory problems with large databases
 756     //        (databases may easily have >1 Mio source records)
 757     //  * it does not make sense to search for existing sources and then clone them
 758     //    we need to search for existing references instead and use them (if exist)
 759     //    for our new source.
 760     protected IdentifiableSource getIdentifiableSource(Reference reference, String citationDetail) {
 761
 762         IdentifiableSource sour = IdentifiableSource.NewInstance(OriginalSourceType.Import,null,null, reference,citationDetail);
 763         return sour;
 764     }
 765
 766     /**
 767      * Add the hierarchy for a Taxon(add higher taxa)
 768      * @param classification
 769      * @param taxon: a taxon to add as a node
 770      * @param state: the ABCD import state
 771      */
 772     protected void addParentTaxon(Taxon taxon, STATE state, boolean preferredFlag, Classification classification){
 773         INonViralName  nvname = taxon.getName();
 774         Rank rank = nvname.getRank();
 775         Taxon genus =null;
 776         Taxon subgenus =null;
 777         Taxon species = null;
 778         Taxon subspecies = null;
 779         Taxon parent = null;
 780         if(rank!=null){
 781             if (rank.isLowerThan(RankClass.Genus)){
 782                 String genusOrUninomial = nvname.getGenusOrUninomial();
 783                 TaxonName taxonName = getOrCreateTaxonName(genusOrUninomial, Rank.GENUS(), preferredFlag, state, -1);
 784                 genus = getOrCreateTaxonForName(taxonName, state);
 785                 if (genus == null){
 786                     logger.debug("The genus should not be null " + taxonName);
 787                 }
 788                 if (preferredFlag) {
 789                     parent = linkParentChildNode(null, genus, classification, state);
 790                 }
 791
 792             }
 793             if (rank.isLower(Rank.SUBGENUS())){
 794                 String prefix = nvname.getGenusOrUninomial();
 795                 String name = nvname.getInfraGenericEpithet();
 796                 if (name != null){
 797                     TaxonName taxonName = getOrCreateTaxonName(prefix+" "+name, Rank.SUBGENUS(), preferredFlag, state, -1);
 798                     subgenus = getOrCreateTaxonForName(taxonName, state);
 799                     if (preferredFlag) {
 800                         parent = linkParentChildNode(genus, subgenus, classification, state);
 801                     }
 802                 }
 803             }
 804             if (rank.isLowerThan(RankClass.Species)){
 805                 if (subgenus!=null){
 806                     String prefix = nvname.getGenusOrUninomial();
 807                     String name = nvname.getInfraGenericEpithet();
 808                     String spe = nvname.getSpecificEpithet();
 809                     if (spe != null){
 810                         TaxonName taxonName = getOrCreateTaxonName(prefix+" "+name+" "+spe, Rank.SPECIES(), preferredFlag, state, -1);
 811                         species = getOrCreateTaxonForName(taxonName, state);
 812                         if (preferredFlag) {
 813                             parent = linkParentChildNode(subgenus, species, classification, state);
 814                         }
 815                     }
 816                 }
 817                 else{
 818                     String prefix = nvname.getGenusOrUninomial();
 819                     String name = nvname.getSpecificEpithet();
 820                     if (name != null){
 821                         TaxonName taxonName = getOrCreateTaxonName(prefix+" "+name, Rank.SPECIES(), preferredFlag, state, -1);
 822                         species = getOrCreateTaxonForName(taxonName, state);
 823                         if (preferredFlag) {
 824                             parent = linkParentChildNode(genus, species, classification, state);
 825                         }
 826                     }
 827                 }
 828             }
 829             if (rank.isLower(Rank.INFRASPECIES())){
 830                 TaxonName taxonName = getOrCreateTaxonName(nvname.getFullTitleCache(), Rank.SUBSPECIES(), preferredFlag, state, -1);
 831                 subspecies = getOrCreateTaxonForName(taxonName, state);
 832                 if (preferredFlag) {
 833                     parent = linkParentChildNode(species, subspecies, classification, state);
 834                 }
 835             }
 836         }else{
 837             //handle cf. and aff. taxa
 838             String genusEpithet = null;
 839             if (nvname.getTitleCache().contains("cf.")){
 840                 genusEpithet = nvname.getTitleCache().substring(0, nvname.getTitleCache().indexOf("cf."));
 841             } else if (nvname.getTitleCache().contains("aff.")){
 842                 genusEpithet = nvname.getTitleCache().substring(0, nvname.getTitleCache().indexOf("aff."));
 843             }
 844             if (genusEpithet != null){
 845                     genusEpithet = genusEpithet.trim();
 846                     TaxonName taxonName = null;
 847                     if (genusEpithet.contains(" ")){
 848                         taxonName = getOrCreateTaxonName(genusEpithet, Rank.SPECIES(), preferredFlag, state, -1);
 849                     }else{
 850                         taxonName = getOrCreateTaxonName(genusEpithet, Rank.GENUS(), preferredFlag, state, -1);
 851                     }
 852                     genus = getOrCreateTaxonForName(taxonName, state);
 853                 if (genus == null){
 854                     logger.debug("The genus should not be null " + taxonName);
 855                 }
 856                 if (preferredFlag) {
 857                     parent = linkParentChildNode(null, genus, classification, state);
 858                 }
 859             }
 860         }
 861         if (preferredFlag && parent!=taxon ) {
 862             linkParentChildNode(parent, taxon, classification, state);
 863         }
 864     }
 865
 866     /**
 867      * Link a parent to a child and save it in the current classification
 868      * @param parent: the higher Taxon
 869      * @param child : the lower (or current) Taxon
 870      * return the Taxon from the new created Node
 871      * @param classification
 872      * @param state
 873      */
 874     protected Taxon linkParentChildNode(Taxon parent, Taxon child, Classification classification, STATE state) {
 875         TaxonNode node =null;
 876         List<String> propertyPaths = new ArrayList<>();
 877         propertyPaths.add("childNodes");
 878         if (parent != null) {
 879
 880             parent = (Taxon) getTaxonService().load(parent.getUuid(), propertyPaths);
 881             child = (Taxon) getTaxonService().load(child.getUuid(), propertyPaths);
 882             //here we do not have to check if the taxon nodes already exists
 883             //this is done by classification.addParentChild()
 884             //do not add child node if it already exists
 885             if(hasTaxonNodeInClassification(child, classification)){
 886                 return child;
 887             }
 888             else{
 889                 node = classification.addParentChild(parent, child, state.getRef(), "");
 890                 save(node, state);
 891             }
 892         }
 893         else {
 894             if (child == null){
 895                 logger.debug("The child should not be null!");
 896             }
 897             child = (Taxon) getTaxonService().find(child.getUuid());
 898             //do not add child node if it already exists
 899             if(hasTaxonNodeInClassification(child, classification)){
 900                 return child;
 901             }
 902             else{
 903                 node = classification.addChildTaxon(child, state.getRef(), null);
 904                 save(node, state);
 905             }
 906         }
 907         if(node!=null){
 908             state.getReport().addTaxonNode(node);
 909             return node.getTaxon();
 910         }
 911         String message = "Could not create taxon node for " +child;
 912         state.getReport().addInfoMessage(message);
 913         logger.warn(message);
 914         return null;
 915     }
 916
 917     protected Taxon getOrCreateTaxonForName(TaxonName taxonName, STATE state){
 918         if (taxonName != null){
 919                 Set<Taxon> acceptedTaxa = taxonName.getTaxa();
 920                 if(acceptedTaxa.size()>0){
 921                     Taxon firstAcceptedTaxon = acceptedTaxa.iterator().next();
 922                     if(acceptedTaxa.size()>1){
 923                         String message = "More than one accepted taxon was found for taxon name: "
 924                                 + taxonName.getTitleCache() + "!\n" + firstAcceptedTaxon + "was chosen for "+state.getDerivedUnitBase();
 925                         state.getReport().addInfoMessage(message);
 926                         logger.warn(message);
 927                     }
 928                     else{
 929                         return firstAcceptedTaxon;
 930                     }
 931                 }
 932                 else{
 933                     @SuppressWarnings("rawtypes")
 934                 Set<TaxonBase> taxonAndSynonyms = taxonName.getTaxonBases();
 935                     for (TaxonBase<?> taxonBase : taxonAndSynonyms) {
 936                         if(taxonBase.isInstanceOf(Synonym.class)){
 937                             Synonym synonym = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
 938                             Taxon acceptedTaxonOfSynonym = synonym.getAcceptedTaxon();
 939                             if(acceptedTaxonOfSynonym == null){
 940                                 String message = "No accepted taxon could be found for taxon name: "
 941                                         + taxonName.getTitleCache()
 942                                         + "!";
 943                                 state.getReport().addInfoMessage(message);
 944                                 logger.warn(message);
 945                             }
 946                             else{
 947                                 return acceptedTaxonOfSynonym;
 948                             }
 949                         }
 950                     }
 951                 }
 952                 Taxon taxon = Taxon.NewInstance(taxonName, state.getRef());
 953                 save(taxon, state);
 954                 state.getReport().addTaxon(taxon);
 955                 logger.info("Created new taxon "+ taxon);
 956                 return taxon;
 957         }
 958         return null;
 959     }
 960
 961     private boolean hasTaxonNodeInClassification(Taxon taxon, Classification classification){
 962         if(taxon.getTaxonNodes()!=null){
 963             for (TaxonNode node : taxon.getTaxonNodes()){
 964                 if(node.getClassification().equals(classification)){
 965                     return true;
 966                 }
 967             }
 968         }
 969         return false;
 970     }
 971
 972     /**
 973      * HandleIdentifications : get the scientific names present in the ABCD
 974      * document and store link them with the observation/specimen data
 975      * @param state: the current ABCD import state
 976      * @param derivedUnitFacade : the current derivedunitfacade
 977      */
 978     protected void handleIdentifications(STATE state, DerivedUnitFacade derivedUnitFacade) {
 979
 980         SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
 981
 982         String scientificName = "";
 983         boolean preferredFlag = false;
 984
 985         if (state.getDataHolder().getNomenclatureCode() == ""){
 986             if (config.getNomenclaturalCode() != null){
 987                 if (config.getNomenclaturalCode() != null){
 988                     state.getDataHolder().setNomenclatureCode(config.getNomenclaturalCode().toString());
 989
 990                 }
 991             }
 992         }
 993
 994         for (int i = 0; i < state.getDataHolder().getIdentificationList().size(); i++) {
 995             Identification identification = state.getDataHolder().getIdentificationList().get(i);
 996             scientificName = identification.getScientificName().replaceAll(" et ", " & ");
 997
 998             String preferred = identification.getPreferred();
 999             preferredFlag = false;
1000             if (preferred != null || state.getDataHolder().getIdentificationList().size()==1){
1001                 if (state.getDataHolder().getIdentificationList().size()==1){
1002                     preferredFlag = true;
1003                 }else if (preferred != null && (preferred.equals("1") || preferred.toLowerCase().indexOf("true") != -1) ) {
1004                         preferredFlag = true;
1005                     }
1006
1007             }
1008             if (identification.getCode() != null){
1009                     if (identification.getCode().indexOf(':') != -1) {
1010                         state.getDataHolder().setNomenclatureCode(identification.getCode().split(COLON)[1]);
1011                     }
1012                     else{
1013                         state.getDataHolder().setNomenclatureCode(identification.getCode());
1014                     }
1015             }
1016             TaxonName taxonName = getOrCreateTaxonName(scientificName, null, preferredFlag, state, i);
1017             Taxon taxon = getOrCreateTaxonForName(taxonName, state);
1018             addTaxonNode(taxon, state,preferredFlag);
1019             linkDeterminationEvent(state, taxon, preferredFlag, derivedUnitFacade, identification.getIdentifier(), identification.getDate(), identification.getModifier());
1020         }
1021     }
1022
1023     /**
1024      * @param taxon : a taxon to add as a node
1025      * @param state : the ABCD import state
1026      */
1027     protected void addTaxonNode(Taxon taxon, STATE state, boolean preferredFlag) {
1028         SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
1029         logger.info("link taxon to a taxonNode "+taxon.getTitleCache());
1030         //only add nodes if not already existing in current classification or default classification
1031
1032         //check if node exists in current classification
1033         //NOTE: we cannot use hasTaxonNodeInClassification() here because we are first creating it here
1034         if (!existsInClassification(taxon,state.getClassification(), state)){
1035             if(config.isMoveNewTaxaToDefaultClassification()){
1036                 //check if node exists in default classification
1037                 if (!existsInClassification(taxon, state.getDefaultClassification(true), state)){
1038                     addParentTaxon(taxon, state, preferredFlag, state.getDefaultClassification(true));
1039                 }
1040             }else{
1041                 //add non-existing taxon to current classification
1042                 addParentTaxon(taxon, state, preferredFlag, state.getClassification());
1043             }
1044
1045         }
1046     }
1047
1048
1049     private boolean existsInClassification(Taxon taxon, Classification classification, STATE state){
1050         boolean exist = false;
1051         ICdmRepository cdmAppController = state.getConfig().getCdmAppController();
1052         if(cdmAppController==null){
1053             cdmAppController = this;
1054         }
1055         if (classification != null){
1056             if (!taxon.getTaxonNodes().isEmpty()){
1057                 for (TaxonNode node:taxon.getTaxonNodes()){
1058                     if (node.getClassification().equals(classification)){
1059                         return true;
1060                     }
1061                 }
1062             }
1063 // we do not need this because we already searched for taxa in db in the previous steps
1064 //              List<UuidAndTitleCache<TaxonNode>> uuidAndTitleCacheOfAllTaxa = cdmAppController.getClassificationService().getTaxonNodeUuidAndTitleCacheOfAcceptedTaxaByClassification(classification.getUuid());
1065 //              if (uuidAndTitleCacheOfAllTaxa != null){
1066 //                      for (UuidAndTitleCache p : uuidAndTitleCacheOfAllTaxa){
1067 //                          try{
1068 //                              if(p.getTitleCache().equals(taxon.getTitleCache())) {
1069 //                                  exist = true;
1070 //                              }
1071 //                          }
1072 //                          catch(Exception e){
1073 //                              logger.warn("TaxonNode doesn't seem to have a taxon");
1074 //                          }
1075 //                      }
1076 //              }
1077         }
1078         return exist;
1079     }
1080
1081     /**
1082      * join DeterminationEvent to the Taxon Object
1083      * @param state : the ABCD import state
1084      * @param taxon: the current Taxon
1085      * @param preferredFlag :if the current name is preferred
1086      * @param derivedFacade : the derived Unit Facade
1087      */
1088     @SuppressWarnings("rawtypes")
1089     protected void linkDeterminationEvent(STATE state, Taxon taxon, boolean preferredFlag,  DerivedUnitFacade derivedFacade, String identifierStr, String dateStr, String modifier) {
1090         SpecimenImportConfiguratorBase config = state.getConfig();
1091         if (logger.isDebugEnabled()){
1092             logger.info("start linkdetermination with taxon:" + taxon.getUuid()+", "+taxon);
1093         }
1094
1095         DeterminationEvent determinationEvent = DeterminationEvent.NewInstance();
1096         //determinationEvent.setTaxon(taxon);
1097         determinationEvent.setTaxonName(taxon.getName());
1098         determinationEvent.setPreferredFlag(preferredFlag);
1099
1100
1101         determinationEvent.setIdentifiedUnit(state.getDerivedUnitBase());
1102         if (state.getPersonStore().get(identifierStr) != null){
1103             determinationEvent.setActor((AgentBase)state.getPersonStore().get(identifierStr));
1104         } else if (identifierStr != null){
1105             Person identifier = Person.NewTitledInstance(identifierStr);
1106             determinationEvent.setActor(identifier);
1107         }
1108         if (dateStr != null){
1109             determinationEvent.setTimeperiod(TimePeriodParser.parseString(dateStr));
1110         }
1111         if (modifier != null){
1112             if (modifier.equals("cf.")){
1113                 determinationEvent.setModifier(DefinedTerm.DETERMINATION_MODIFIER_CONFER());
1114             }else if (modifier.equals("aff.")){
1115                 determinationEvent.setModifier(DefinedTerm.DETERMINATION_MODIFIER_AFFINIS());
1116             }
1117         }
1118         if (config.isAddDeterminations()) {
1119             state.getDerivedUnitBase().addDetermination(determinationEvent);
1120         }
1121
1122
1123         if (logger.isDebugEnabled()){
1124             logger.debug("NB TYPES INFO: "+ state.getDataHolder().getStatusList().size());
1125         }
1126         for (SpecimenTypeDesignationStatus specimenTypeDesignationstatus : state.getDataHolder().getStatusList()) {
1127             if (specimenTypeDesignationstatus != null) {
1128                 if (logger.isDebugEnabled()){
1129                     logger.debug("specimenTypeDesignationstatus :"+ specimenTypeDesignationstatus);
1130                 }
1131
1132                 ICdmRepository cdmAppController = config.getCdmAppController();
1133                 if(cdmAppController == null){
1134                     cdmAppController = this;
1135                 }
1136                 specimenTypeDesignationstatus = HibernateProxyHelper.deproxy(cdmAppController.getTermService().find(specimenTypeDesignationstatus.getUuid()), SpecimenTypeDesignationStatus.class);
1137                 //Designation
1138                 TaxonName name = taxon.getName();
1139                 SpecimenTypeDesignation designation = SpecimenTypeDesignation.NewInstance();
1140
1141                 designation.setTypeStatus(specimenTypeDesignationstatus);
1142                 designation.setTypeSpecimen(state.getDerivedUnitBase());
1143                 name.addTypeDesignation(designation, false);
1144             }
1145         }
1146         save(state.getDerivedUnitBase(), state);
1147
1148         for (String[] fullReference : state.getDataHolder().getReferenceList()) {
1149
1150
1151             String strReference=fullReference[0];
1152             String citationDetail = fullReference[1];
1153             String citationURL = fullReference[2];
1154             List<Reference> references = getReferenceService().listByTitleWithRestrictions(Reference.class, "strReference", MatchMode.EXACT, null, null, null, null, null);
1155
1156             if (!references.isEmpty()){
1157                 Reference reference = null;
1158                 for (Reference refe: references) {
1159                     if (refe.getTitleCache().equalsIgnoreCase(strReference)) {
1160                         reference =refe;
1161                         break;
1162                     }
1163                 }
1164                 if (reference ==null){
1165                     reference = ReferenceFactory.newGeneric();
1166                     reference.setTitleCache(strReference, true);
1167                     save(reference, state);
1168                 }
1169                 determinationEvent.addReference(reference);
1170             }
1171         }
1172         save(state.getDerivedUnitBase(), state);
1173
1174         if (config.isAddIndividualsAssociations() && preferredFlag) {
1175             //do not add IndividualsAssociation to non-preferred taxa
1176             if (logger.isDebugEnabled()){
1177                 logger.debug("isDoCreateIndividualsAssociations");
1178             }
1179
1180             makeIndividualsAssociation(state, taxon, determinationEvent);
1181
1182             save(state.getDerivedUnitBase(), state);
1183         }
1184     }
1185
1186     /**
1187      * create and link each association (specimen, observation..) to the accepted taxon
1188      * @param state : the ABCD import state
1189      * @param taxon: the current Taxon
1190      * @param determinationEvent:the determinationevent
1191      */
1192     protected void makeIndividualsAssociation(STATE state, Taxon taxon, DeterminationEvent determinationEvent) {
1193         SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
1194         SpecimenUserInteraction sui = config.getSpecimenUserInteraction();
1195
1196         if (logger.isDebugEnabled()){
1197             logger.info("MAKE INDIVIDUALS ASSOCIATION");
1198         }
1199
1200         TaxonDescription taxonDescription = null;
1201         Set<TaxonDescription> descriptions= taxon.getDescriptions();
1202         if (state.getIndividualsAssociationDescriptionPerTaxon(taxon.getUuid()) != null){
1203             taxonDescription = state.getIndividualsAssociationDescriptionPerTaxon(taxon.getUuid());
1204         }
1205        if (taxonDescription == null && !descriptions.isEmpty() && state.getConfig().isReuseExistingDescriptiveGroups()){
1206            for (TaxonDescription desc: descriptions){
1207                if (desc.getTypes().contains(DescriptionType.INDIVIDUALS_ASSOCIATION)){
1208                    taxonDescription = desc;
1209                    break;
1210                }
1211            }
1212        }
1213
1214        if (taxonDescription == null){
1215             taxonDescription = TaxonDescription.NewInstance(taxon, false);
1216             taxonDescription.setTypes(EnumSet.of(DescriptionType.INDIVIDUALS_ASSOCIATION));
1217             if(sourceNotLinkedToElement(taxonDescription,state.getRef(),null)) {
1218                 taxonDescription.addSource(OriginalSourceType.Import, null, null, state.getRef(), null);
1219             }
1220             state.setIndividualsAssociationDescriptionPerTaxon(taxonDescription);
1221             taxon.addDescription(taxonDescription);
1222         }
1223
1224         //PREPARE REFERENCE QUESTIONS
1225
1226         Map<String,OriginalSourceBase> sourceMap = new HashMap<>();
1227
1228         List<IdentifiableSource> issTmp = new ArrayList<>();//getCommonService().list(IdentifiableSource.class, null, null, null, null);
1229         List<DescriptionElementSource> issTmp2 = new ArrayList<>();//getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1230
1231         Set<OriginalSourceBase> osbSet = new HashSet<>();
1232         if(issTmp2!=null) {
1233             osbSet.addAll(issTmp2);
1234         }
1235         if(issTmp!=null) {
1236             osbSet.addAll(issTmp);
1237         }
1238
1239
1240         addToSourceMap(sourceMap, osbSet);
1241
1242
1243         if(sourceNotLinkedToElement(taxonDescription,state.getRef(),null)) {
1244             taxonDescription.addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1245         }
1246
1247         state.setIndividualsAssociationDescriptionPerTaxon(taxonDescription);
1248
1249         IndividualsAssociation indAssociation = IndividualsAssociation.NewInstance();
1250         Feature feature = makeFeature(state.getDerivedUnitBase());
1251         indAssociation.setAssociatedSpecimenOrObservation(state.getDerivedUnitBase());
1252         indAssociation.setFeature(feature);
1253
1254         if(sourceNotLinkedToElement(indAssociation,state.getImportReference(state.getActualAccessPoint()),null)) {
1255             indAssociation.addSource(OriginalSourceType.Import,null, null, state.getImportReference(state.getActualAccessPoint()), null);
1256         }
1257         if(sourceNotLinkedToElement(state.getDerivedUnitBase(), state.getImportReference(state.getActualAccessPoint()),null)) {
1258             state.getDerivedUnitBase().addSource(OriginalSourceType.Import,null, null, state.getImportReference(state.getActualAccessPoint()), null);
1259         }
1260         for (Reference citation : determinationEvent.getReferences()) {
1261             if(sourceNotLinkedToElement(indAssociation,citation,null))
1262             {
1263                 indAssociation.addSource(DescriptionElementSource.NewInstance(OriginalSourceType.Import, null, null, citation, null));
1264             }
1265             if(sourceNotLinkedToElement(state.getDerivedUnitBase(), state.getImportReference(state.getActualAccessPoint()),null)) {
1266                 state.getDerivedUnitBase().addSource(OriginalSourceType.Import,null, null, state.getImportReference(state.getActualAccessPoint()), null);
1267             }
1268         }
1269
1270         taxonDescription.addElement(indAssociation);
1271
1272         save(taxonDescription, state);
1273         save(taxon, state);
1274         state.getReport().addDerivate(state.getDerivedUnitBase(), config);
1275         state.getReport().addIndividualAssociation(taxon, state.getDataHolder().getUnitID(), state.getDerivedUnitBase());
1276     }
1277
1278     private boolean sourceNotLinkedToElement(DerivedUnit derivedUnitBase2, Reference b, String d) {
1279         Set<IdentifiableSource> linkedSources = derivedUnitBase2.getSources();
1280         for (IdentifiableSource is:linkedSources){
1281             Reference a = is.getCitation();
1282             String c = is.getCitationMicroReference();
1283
1284             boolean refMatch=false;
1285             boolean microMatch=false;
1286
1287             try{
1288                 if (a==null && b==null) {
1289                     refMatch=true;
1290                 }
1291                 if (a!=null && b!=null) {
1292                     if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1293                         refMatch=true;
1294                     }
1295                 }
1296             }catch(Exception e){}
1297
1298             try{
1299                 if (c==null && d==null) {
1300                     microMatch=true;
1301                 }
1302                 if(c!=null && d!=null) {
1303                     if(c.equalsIgnoreCase(d)) {
1304                         microMatch=true;
1305                     }
1306                 }
1307             }
1308             catch(Exception e){}
1309
1310             if (microMatch && refMatch) {
1311                 return false;
1312             }
1313         }
1314         return true;
1315     }
1316
1317     private <T extends OriginalSourceBase> boolean  sourceNotLinkedToElement(
1318             ISourceable<T> sourcable, Reference reference, String microReference) {
1319
1320         Set<T> linkedSources = sourcable.getSources();
1321         for (T is:linkedSources){
1322             Reference unitReference = is.getCitation();
1323             String unitMicroReference = is.getCitationMicroReference();
1324
1325             boolean refMatch=false;
1326             boolean microMatch=false;
1327
1328             try{
1329                 if (unitReference==null && reference==null) {
1330                     refMatch=true;
1331                 }
1332                 if (unitReference!=null && reference!=null) {
1333                     if (unitReference.getTitleCache().equalsIgnoreCase(reference.getTitleCache())) {
1334                         refMatch=true;
1335                     }
1336                 }
1337             }catch(Exception e){}
1338
1339             try{
1340                 if (unitMicroReference==null && microReference==null) {
1341                     microMatch=true;
1342                 }
1343                 if(unitMicroReference!=null && microReference!=null) {
1344                     if(unitMicroReference.equalsIgnoreCase(microReference)) {
1345                         microMatch=true;
1346                     }
1347                 }
1348             }
1349             catch(Exception e){}
1350
1351             if (microMatch && refMatch) {
1352                 return false;
1353             }
1354         }
1355         return true;
1356     }
1357
1358     /**
1359      * look for the Feature object (FieldObs, Specimen,...)
1360      * @param unit : a specimen or obersvation base
1361      * @return the corresponding Feature
1362      */
1363     private Feature makeFeature(SpecimenOrObservationBase<?> unit) {
1364         SpecimenOrObservationType type = unit.getRecordBasis();
1365
1366         if (type.isFeatureObservation()){
1367             return Feature.OBSERVATION();
1368         }else if (type.isFeatureSpecimen()){
1369             return Feature.SPECIMEN();
1370         }else if (type == SpecimenOrObservationType.DerivedUnit){
1371             return Feature.OBSERVATION();
1372             //            return getFeature("Specimen or observation");
1373         }else{
1374             String message = "Unhandled record basis '%s' for defining individuals association feature type. Use default.";
1375             logger.warn(String.format(message, type.getLabel()));
1376             return Feature.OBSERVATION();
1377             //            return getFeature("Specimen or observation");
1378         }
1379     }
1380
1381     protected void addToSourceMap(Map<String, OriginalSourceBase> sourceMap, Set<OriginalSourceBase> osbSet) {
1382         for( OriginalSourceBase osb:osbSet) {
1383             if(osb.getCitation()!=null && osb.getCitationMicroReference() !=null  && !osb.getCitationMicroReference().isEmpty()) {
1384                 try{
1385                     sourceMap.put(osb.getCitation().getTitleCache()+ "---"+osb.getCitationMicroReference(),osb);
1386                 }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1387             } else if(osb.getCitation()!=null){
1388                 try{
1389                     sourceMap.put(osb.getCitation().getTitleCache(),osb);
1390                 }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1391             }
1392         }
1393     }
1394 }