cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/SpecimenImportBase.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.specimen;
  11
  12
  13 import java.util.ArrayList;
  14 import java.util.EnumSet;
  15 import java.util.HashMap;
  16 import java.util.HashSet;
  17 import java.util.List;
  18 import java.util.Map;
  19 import java.util.Set;
  20 import java.util.UUID;
  21
  22 import org.apache.log4j.Logger;
  23
  24 import eu.etaxonomy.cdm.api.application.ICdmRepository;
  25 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
  26 import eu.etaxonomy.cdm.api.service.config.FindOccurrencesConfigurator;
  27 import eu.etaxonomy.cdm.api.service.pager.Pager;
  28 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  29 import eu.etaxonomy.cdm.io.common.CdmImportBase;
  30 import eu.etaxonomy.cdm.io.common.IImportConfigurator;
  31 import eu.etaxonomy.cdm.io.specimen.abcd206.in.Identification;
  32 import eu.etaxonomy.cdm.io.specimen.abcd206.in.SpecimenImportReport;
  33 import eu.etaxonomy.cdm.model.agent.AgentBase;
  34 import eu.etaxonomy.cdm.model.agent.Institution;
  35 import eu.etaxonomy.cdm.model.agent.Person;
  36 import eu.etaxonomy.cdm.model.agent.Team;
  37 import eu.etaxonomy.cdm.model.common.CdmBase;
  38 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  39 import eu.etaxonomy.cdm.model.common.LanguageString;
  40 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  41 import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
  42 import eu.etaxonomy.cdm.model.description.DescriptionType;
  43 import eu.etaxonomy.cdm.model.description.Feature;
  44 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  45 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  46 import eu.etaxonomy.cdm.model.name.INonViralName;
  47 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  48 import eu.etaxonomy.cdm.model.name.Rank;
  49 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
  50 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
  51 import eu.etaxonomy.cdm.model.name.TaxonName;
  52 import eu.etaxonomy.cdm.model.name.TaxonNameFactory;
  53 import eu.etaxonomy.cdm.model.occurrence.Collection;
  54 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  55 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
  56 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
  57 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationType;
  58 import eu.etaxonomy.cdm.model.reference.ISourceable;
  59 import eu.etaxonomy.cdm.model.reference.OriginalSourceBase;
  60 import eu.etaxonomy.cdm.model.reference.OriginalSourceType;
  61 import eu.etaxonomy.cdm.model.reference.Reference;
  62 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  63 import eu.etaxonomy.cdm.model.taxon.Classification;
  64 import eu.etaxonomy.cdm.model.taxon.Synonym;
  65 import eu.etaxonomy.cdm.model.taxon.Taxon;
  66 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  67 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  68 import eu.etaxonomy.cdm.model.term.DefinedTerm;
  69 import eu.etaxonomy.cdm.persistence.query.MatchMode;
  70 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
  71 import eu.etaxonomy.cdm.strategy.parser.ParserProblem;
  72 import eu.etaxonomy.cdm.strategy.parser.TimePeriodParser;
  73
  74
  75 /**
  76  * @author p.kelbert
  77  * @since 20.10.2008
  78  */
  79 public abstract class SpecimenImportBase<CONFIG extends IImportConfigurator, STATE extends SpecimenImportStateBase>
  80         extends CdmImportBase<CONFIG, STATE> {
  81
  82     private static final long serialVersionUID = 4423065367998125678L;
  83     private static final Logger logger = Logger.getLogger(SpecimenImportBase.class);
  84
  85         protected static final UUID SPECIMEN_SCAN_TERM = UUID.fromString("acda15be-c0e2-4ea8-8783-b9b0c4ad7f03");
  86
  87         private static final String COLON = ":";
  88
  89         protected Map<String, DefinedTerm> kindOfUnitsMap;
  90
  91
  92
  93         @Override
  94     protected abstract void doInvoke(STATE state);
  95
  96         /**
  97          * Handle a single unit
  98          * @param state
  99          * @param item
 100          */
 101         protected abstract void handleSingleUnit(STATE state, Object item) ;
 102
 103
 104
 105         protected TaxonName getOrCreateTaxonName(String scientificName, Rank rank, boolean preferredFlag, STATE state, int unitIndexInAbcdFile){
 106             TaxonName taxonName = null;
 107         SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
 108
 109         //check atomised name data for rank
 110         //new name will be created
 111         TaxonName atomisedTaxonName = null;
 112         if (rank==null && unitIndexInAbcdFile>=0 && ((state.getDataHolder().getAtomisedIdentificationList() != null && !state.getDataHolder().getAtomisedIdentificationList().isEmpty())|| state.getDataHolder().getAtomisedIdentificationList().size() > 0)) {
 113             atomisedTaxonName = setTaxonNameByType(state.getDataHolder().getAtomisedIdentificationList().get(unitIndexInAbcdFile), scientificName, state);
 114             if(atomisedTaxonName!=null){
 115                 rank = atomisedTaxonName.getRank();
 116             }
 117         }
 118         if(config.isReuseExistingTaxaWhenPossible()){
 119             TaxonName parsedName = atomisedTaxonName;
 120             if(parsedName==null){
 121
 122                 parsedName = parseScientificName(scientificName, state, state.getReport(), rank);
 123
 124             }
 125             atomisedTaxonName = parsedName;
 126             if(config.isIgnoreAuthorship() && parsedName!=null){// && preferredFlag){
 127                 // do not ignore authorship for non-preferred names because they need
 128                 // to be created for the determination history
 129                 String nameCache = TaxonName.castAndDeproxy(parsedName).getNameCache();
 130                 List<TaxonName> names = getNameService().findNamesByNameCache(nameCache, MatchMode.EXACT, null);
 131                 if (!names.isEmpty()){
 132                      taxonName = getBestMatchingName(scientificName, new ArrayList<>(names), state);
 133                 }
 134                 if (taxonName == null && !names.isEmpty()){
 135                     taxonName = names.get(0);
 136                 }
 137
 138             } else {
 139                 //search for existing names
 140                 List<TaxonName> names = getNameService().listByTitleWithRestrictions(TaxonName.class, scientificName, MatchMode.EXACT, null, null, null, null, null);
 141                 taxonName = getBestMatchingName(scientificName, names, state);
 142                 //still nothing found -> try with the atomised name full title cache
 143                 if(taxonName==null && atomisedTaxonName!=null){
 144                     names = getNameService().listByTitleWithRestrictions(TaxonName.class, atomisedTaxonName.getFullTitleCache(), MatchMode.EXACT, null, null, null, null, null);
 145                     taxonName = getBestMatchingName(atomisedTaxonName.getTitleCache(), names, state);
 146                     //still nothing found -> try with the atomised name title cache
 147                     if(taxonName==null){
 148                         names = getNameService().listByTitleWithRestrictions(TaxonName.class, atomisedTaxonName.getTitleCache(), MatchMode.EXACT, null, null, null, null, null);
 149                         taxonName = getBestMatchingName(atomisedTaxonName.getTitleCache(), names, state);
 150                     }
 151                 }
 152
 153             }
 154
 155         }
 156
 157         if(taxonName==null && atomisedTaxonName!=null){
 158             taxonName = atomisedTaxonName;
 159             state.getReport().addName(taxonName);
 160             logger.info("Created new taxon name "+taxonName);
 161             if(taxonName.hasProblem()){
 162                 state.getReport().addInfoMessage(String.format("Created %s with parsing problems", taxonName));
 163             }
 164             if(!atomisedTaxonName.getTitleCache().equals(scientificName)){
 165                 state.getReport().addInfoMessage(String.format("Taxon %s was parsed as %s", scientificName, atomisedTaxonName.getTitleCache()));
 166             }
 167         }
 168         else if(taxonName==null){
 169             //create new taxon name
 170
 171             if (state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNAFP)){
 172                 taxonName = TaxonNameFactory.NewBotanicalInstance(rank);
 173             }else if (state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICZN)){
 174                 taxonName = TaxonNameFactory.NewZoologicalInstance(rank);
 175             }else{
 176                 taxonName = TaxonNameFactory.NewNonViralInstance(rank);
 177             }
 178             taxonName.setFullTitleCache(scientificName,true);
 179             taxonName.setTitleCache(scientificName, true);
 180             state.getReport().addName(taxonName);
 181             logger.info("Created new taxon name "+taxonName);
 182         }
 183         if (taxonName != null){
 184             state.names.put(taxonName.getNameCache(), taxonName);
 185         }
 186         if(!taxonName.isPersited()) {
 187             save(taxonName, state);
 188         }
 189         return taxonName;
 190     }
 191
 192          protected TaxonName getBestMatchingName(String scientificName, java.util.Collection<TaxonName> names, STATE state){
 193                 Set<TaxonName> namesWithAcceptedTaxa = new HashSet<>();
 194                 List<TaxonName> namesWithAcceptedTaxaInClassification = new ArrayList<>();
 195                 for (TaxonName name : names) {
 196                     if(!name.getTaxa().isEmpty()){
 197                         Set<Taxon> taxa = name.getTaxa();
 198                         for (Taxon taxon:taxa){
 199                             if (!taxon.getTaxonNodes().isEmpty()){
 200                                 //use only taxa included in a classification
 201                                 for (TaxonNode node:taxon.getTaxonNodes()){
 202                                     if (state.getClassification() != null && node.getClassification().equals(state.getClassification())){
 203                                         namesWithAcceptedTaxaInClassification.add(name);
 204                                     }else {
 205                                         namesWithAcceptedTaxa.add(name);
 206                                     }
 207                                 }
 208
 209                             }
 210                         }
 211
 212                     }
 213                 }
 214                 String message = String.format("More than one taxon name was found for %s, maybe in other classifications!", scientificName);
 215                 //check for names with accepted taxa in classification
 216                 if(namesWithAcceptedTaxaInClassification.size()>0){
 217                 if(namesWithAcceptedTaxaInClassification.size()>1){
 218
 219                     state.getReport().addInfoMessage(message);
 220                     logger.warn(message);
 221                     return null;
 222                 }
 223                 return namesWithAcceptedTaxaInClassification.iterator().next();
 224             }
 225               //check for any names with accepted taxa
 226                 if(namesWithAcceptedTaxa.size()>0){
 227                     if(namesWithAcceptedTaxa.size()>1){
 228
 229                         state.getReport().addInfoMessage(message);
 230                         logger.warn(message);
 231                         return null;
 232                     }
 233                     return namesWithAcceptedTaxa.iterator().next();
 234                 }
 235                 //no names with accepted taxa found -> check accepted taxa of synonyms
 236                 List<Taxon> taxaFromSynonyms = new ArrayList<>();
 237                 for (TaxonName name : names) {
 238                     Set<TaxonBase> taxonBases = name.getTaxonBases();
 239                     for (TaxonBase taxonBase : taxonBases) {
 240                         if(taxonBase.isInstanceOf(Synonym.class)){
 241                             Synonym synonym = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
 242                             taxaFromSynonyms.add(synonym.getAcceptedTaxon());
 243                         }
 244                     }
 245                 }
 246                 if(taxaFromSynonyms.size()>0){
 247                     if(taxaFromSynonyms.size()>1){
 248                         state.getReport().addInfoMessage(message);
 249                         logger.warn(message);
 250                         return null;
 251                     }
 252                     return taxaFromSynonyms.iterator().next().getName();
 253                 }
 254                 //no accepted and no synonyms -> return one of the names and create a new taxon
 255                 if (names.isEmpty()){
 256                     return null;
 257                 }else{
 258                     return names.iterator().next();
 259                 }
 260             }
 261          /**
 262              * Parse automatically the scientific name
 263              * @param scientificName the scientific name to parse
 264              * @param state the current import state
 265              * @param report the import report
 266              * @return a parsed name
 267              */
 268
 269             protected TaxonName parseScientificName(String scientificName, STATE state, SpecimenImportReport report, Rank rank) {
 270
 271                 NonViralNameParserImpl nvnpi = NonViralNameParserImpl.NewInstance();
 272                 TaxonName taxonName = null;
 273                 boolean problem = false;
 274
 275                 if (logger.isDebugEnabled()){
 276                     logger.debug("parseScientificName " + state.getDataHolder().getNomenclatureCode().toString());
 277                 }
 278
 279                 if (state.getDataHolder().getNomenclatureCode() != null && (state.getDataHolder().getNomenclatureCode().toString().equals("Zoological") || state.getDataHolder().getNomenclatureCode().toString().contains("ICZN"))) {
 280                     taxonName = (TaxonName)nvnpi.parseFullName(scientificName, NomenclaturalCode.ICZN, rank);
 281                     if (taxonName.hasProblem()) {
 282                         problem = true;
 283                     }
 284                 }
 285                 else if (state.getDataHolder().getNomenclatureCode() != null && (state.getDataHolder().getNomenclatureCode().toString().equals("Botanical") || state.getDataHolder().getNomenclatureCode().toString().contains("ICBN")  || state.getDataHolder().getNomenclatureCode().toString().contains("ICNAFP"))) {
 286                     taxonName = (TaxonName)nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNAFP, rank);
 287                     if (taxonName.hasProblem()) {
 288                         problem = true;
 289                     }
 290                 }
 291                 else if (state.getDataHolder().getNomenclatureCode() != null && (state.getDataHolder().getNomenclatureCode().toString().equals("Bacterial") || state.getDataHolder().getNomenclatureCode().toString().contains("ICBN"))) {
 292                     taxonName = (TaxonName)nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNB, rank);
 293                     if (taxonName.hasProblem()) {
 294                         problem = true;
 295                     }
 296                 }
 297                 else if (state.getDataHolder().getNomenclatureCode() != null && (state.getDataHolder().getNomenclatureCode().toString().equals("Cultivar") || state.getDataHolder().getNomenclatureCode().toString().contains("ICNCP"))) {
 298                     taxonName = (TaxonName)nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNCP, rank);
 299                     if (taxonName.hasProblem()) {
 300                         problem = true;
 301                     }
 302                 }
 303                 if (problem) {
 304                     String message = String.format("Parsing problems for %s", scientificName);
 305                     if(taxonName!=null){
 306                         for (ParserProblem parserProblem : taxonName.getParsingProblems()) {
 307                             message += "\n\t- "+parserProblem;
 308                         }
 309                     }
 310                     report.addInfoMessage(message);
 311                     logger.info(message);
 312                 }
 313                 return taxonName;
 314
 315             }
 316
 317             /**
 318              * Create the name without automatic parsing, either because it failed, or because the user deactivated it.
 319              * The name is built upon the ABCD fields
 320              * @param atomisedMap : the ABCD atomised fields
 321              * @param fullName : the full scientific name
 322              * @param state
 323              * @return the corresponding Botanical or Zoological or... name
 324              */
 325             protected TaxonName setTaxonNameByType(
 326                     HashMap<String, String> atomisedMap, String fullName, STATE state) {
 327                 boolean problem = false;
 328                 if (logger.isDebugEnabled()){
 329                     logger.debug("settaxonnamebytype " + state.getDataHolder().getNomenclatureCode().toString());
 330                 }
 331
 332                 if (state.getDataHolder().getNomenclatureCode().equals("Zoological") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICZN.getUuid())) {
 333                     TaxonName taxonName = TaxonNameFactory.NewZoologicalInstance(null);
 334                     taxonName.setFullTitleCache(fullName, true);
 335                     taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
 336                     taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
 337                     taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap,"SpeciesEpithet")));
 338                     taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap,"SubspeciesEpithet")));
 339
 340                     if (taxonName.getGenusOrUninomial() != null){
 341                         taxonName.setRank(Rank.GENUS());
 342                     }
 343
 344                     if (taxonName.getInfraGenericEpithet() != null){
 345                         taxonName.setRank(Rank.SUBGENUS());
 346                     }
 347
 348                     if (taxonName.getSpecificEpithet() != null){
 349                         taxonName.setRank(Rank.SPECIES());
 350                     }
 351
 352                     if (taxonName.getInfraSpecificEpithet() != null){
 353                         taxonName.setRank(Rank.SUBSPECIES());
 354                     }
 355
 356                     Team team = null;
 357                     if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 358                         team = Team.NewInstance();
 359                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
 360                     }
 361                     else {
 362                         if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 363                             team = Team.NewInstance();
 364                             team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
 365                         }
 366                     }
 367                     if (team != null) {
 368                         taxonName.setBasionymAuthorship(team);
 369                     }
 370                     else {
 371                         if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 372                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
 373                         }
 374                         else if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 375                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamAndYear"));
 376                         }
 377                     }
 378                     if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
 379                         team = Team.NewInstance();
 380                         team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
 381                         taxonName.setCombinationAuthorship(team);
 382                     }
 383                     if (taxonName.hasProblem()) {
 384                         logger.info("pb ICZN");
 385                         problem = true;
 386                     }
 387                     else {
 388                         return taxonName;
 389                     }
 390                 }
 391                 else if (state.getDataHolder().getNomenclatureCode().equals("Botanical") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNAFP.getUuid())) {
 392                     TaxonName taxonName = parseScientificName(fullName, state, state.getReport(), null);
 393                     if (taxonName != null){
 394                         return taxonName;
 395                     }
 396                     else{
 397                         taxonName = TaxonNameFactory.NewBotanicalInstance(null);
 398                     }
 399                     taxonName.setFullTitleCache(fullName, true);
 400                     taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
 401                     taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "FirstEpithet")));
 402                     taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "InfraSpeEpithet")));
 403                     try {
 404                         taxonName.setRank(Rank.getRankByName(getFromMap(atomisedMap, "Rank")));
 405                     } catch (Exception e) {
 406                         if (taxonName.getInfraSpecificEpithet() != null){
 407                             taxonName.setRank(Rank.SUBSPECIES());
 408                         }
 409                         else if (taxonName.getSpecificEpithet() != null){
 410                             taxonName.setRank(Rank.SPECIES());
 411                         }
 412                         else if (taxonName.getInfraGenericEpithet() != null){
 413                             taxonName.setRank(Rank.SUBGENUS());
 414                         }
 415                         else if (taxonName.getGenusOrUninomial() != null){
 416                             taxonName.setRank(Rank.GENUS());
 417                         }
 418                     }
 419                     Team team = null;
 420                     if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 421                         team = Team.NewInstance();
 422                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
 423                         taxonName.setBasionymAuthorship(team);
 424                     }
 425                     if (getFromMap(atomisedMap, "AuthorTeam") != null) {
 426                         team = Team.NewInstance();
 427                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeam"), true);
 428                         taxonName.setCombinationAuthorship(team);
 429                     }
 430                     if (team == null) {
 431                         if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 432                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
 433                         }
 434                         else if (getFromMap(atomisedMap, "AuthorTeam") != null) {
 435                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeam"));
 436                         }
 437                     }
 438                     if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
 439                         team = Team.NewInstance();
 440                         team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
 441                         taxonName.setCombinationAuthorship(team);
 442                     }
 443                     if (taxonName.hasProblem()) {
 444                         logger.info("pb ICBN");
 445                         problem = true;
 446                     }
 447                     else {
 448                         return taxonName;
 449                     }
 450                 }
 451                 else if (state.getDataHolder().getNomenclatureCode().equals("Bacterial") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNB.getUuid())) {
 452                     TaxonName taxonName = TaxonNameFactory.NewBacterialInstance(null);
 453                     taxonName.setFullTitleCache(fullName, true);
 454                     taxonName.setGenusOrUninomial(getFromMap(atomisedMap, "Genus"));
 455                     taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
 456                     taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "Species")));
 457                     taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "SubspeciesEpithet")));
 458
 459                     if (taxonName.getGenusOrUninomial() != null){
 460                         taxonName.setRank(Rank.GENUS());
 461                     }
 462                     else if (taxonName.getInfraGenericEpithet() != null){
 463                         taxonName.setRank(Rank.SUBGENUS());
 464                     }
 465                     else if (taxonName.getSpecificEpithet() != null){
 466                         taxonName.setRank(Rank.SPECIES());
 467                     }
 468                     else if (taxonName.getInfraSpecificEpithet() != null){
 469                         taxonName.setRank(Rank.SUBSPECIES());
 470                     }
 471
 472                     if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 473                         Team team = Team.NewInstance();
 474                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
 475                         taxonName.setCombinationAuthorship(team);
 476                     }
 477                     if (getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear") != null) {
 478                         Team team = Team.NewInstance();
 479                         team.setTitleCache(getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear"), true);
 480                         taxonName.setBasionymAuthorship(team);
 481                     }
 482                     if (taxonName.hasProblem()) {
 483                         logger.info("pb ICNB");
 484                         problem = true;
 485                     }
 486                     else {
 487                         return taxonName;
 488                     }
 489                 }
 490                 else if (state.getDataHolder().getNomenclatureCode().equals("Cultivar")) {
 491                     TaxonName taxonName = TaxonNameFactory.NewCultivarInstance(null);
 492
 493                     if (taxonName.hasProblem()) {
 494                         logger.info("pb ICNCP");
 495                         problem = true;
 496                     }
 497                     else {
 498                         return taxonName;
 499                     }
 500                     return taxonName;
 501                 }
 502
 503                 if (problem) {
 504                     logger.info("Problem im setTaxonNameByType ");
 505                     TaxonName taxonName = TaxonNameFactory.NewNonViralInstance(null);
 506                     taxonName.setFullTitleCache(fullName, true);
 507                     return taxonName;
 508                 }
 509                 TaxonName tn = TaxonNameFactory.NewNonViralInstance(null);
 510                 return tn;
 511             }
 512
 513             /**
 514              * Get a formated string from a hashmap
 515              * @param atomisedMap
 516              * @param key
 517              * @return
 518              */
 519             private String getFromMap(HashMap<String, String> atomisedMap, String key) {
 520                 String value = null;
 521                 if (atomisedMap.containsKey(key)) {
 522                     value = atomisedMap.get(key);
 523                 }
 524
 525                 try {
 526                     if (value != null && key.matches(".*Year.*")) {
 527                         value = value.trim();
 528                         if (value.matches("[a-z A-Z ]*[0-9]{4}$")) {
 529                             String tmp = value.split("[0-9]{4}$")[0];
 530                             int year = Integer.parseInt(value.split(tmp)[1]);
 531                             if (year >= 1752) {
 532                                 value = tmp;
 533                             }
 534                             else {
 535                                 value = null;
 536                             }
 537                         }
 538                         else {
 539                             value = null;
 540                         }
 541                     }
 542                 }
 543                 catch (Exception e) {
 544                     value = null;
 545                 }
 546                 return value;
 547             }
 548
 549             /**
 550              * Very fast and dirty implementation to allow handling of transient objects as described in
 551              * https://dev.e-taxonomy.eu/redmine/issues/3726
 552              *
 553              * Not yet complete.
 554              *
 555              * @param cdmBase
 556              * @param state
 557              */
 558             protected UUID save(CdmBase cdmBase, SpecimenImportStateBase state) {
 559                 ICdmRepository cdmRepository = state.getConfig().getCdmAppController();
 560                 if (cdmRepository == null){
 561                     cdmRepository = this;
 562                 }
 563
 564                 if (cdmBase.isInstanceOf(LanguageString.class)){
 565                     return cdmRepository.getTermService().saveLanguageData(CdmBase.deproxy(cdmBase, LanguageString.class));
 566                 }else if (cdmBase.isInstanceOf(SpecimenOrObservationBase.class)){
 567                     SpecimenOrObservationBase specimen = CdmBase.deproxy(cdmBase, SpecimenOrObservationBase.class);
 568
 569                     return cdmRepository.getOccurrenceService().saveOrUpdate(specimen);
 570                 }else if (cdmBase.isInstanceOf(Reference.class)){
 571                     return cdmRepository.getReferenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, Reference.class));
 572                 }else if (cdmBase.isInstanceOf(Classification.class)){
 573                     return cdmRepository.getClassificationService().saveOrUpdate(CdmBase.deproxy(cdmBase, Classification.class));
 574                 }else if (cdmBase.isInstanceOf(AgentBase.class)){
 575                     return cdmRepository.getAgentService().saveOrUpdate(CdmBase.deproxy(cdmBase, AgentBase.class));
 576                 }else if (cdmBase.isInstanceOf(Collection.class)){
 577                     return cdmRepository.getCollectionService().saveOrUpdate(CdmBase.deproxy(cdmBase, Collection.class));
 578                 }else if (cdmBase.isInstanceOf(DescriptionBase.class)){
 579                     DescriptionBase description = CdmBase.deproxy(cdmBase, DescriptionBase.class);
 580
 581                     return cdmRepository.getDescriptionService().saveOrUpdate(description);
 582                 }else if (cdmBase.isInstanceOf(TaxonBase.class)){
 583                     return cdmRepository.getTaxonService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonBase.class));
 584                 }else if (cdmBase.isInstanceOf(TaxonName.class)){
 585                     return cdmRepository.getNameService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonName.class));
 586                 }else if (cdmBase.isInstanceOf(TaxonNode.class)){
 587                 return cdmRepository.getTaxonNodeService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonNode.class));
 588             }else{
 589                     throw new IllegalArgumentException("Class not supported in save method: " + CdmBase.deproxy(cdmBase, CdmBase.class).getClass().getSimpleName());
 590                 }
 591
 592             }
 593
 594
 595             protected SpecimenOrObservationBase findExistingSpecimen(String unitId, SpecimenImportStateBase state){
 596                 ICdmRepository cdmAppController = state.getConfig().getCdmAppController();
 597                 if(cdmAppController==null){
 598                     cdmAppController = this;
 599                 }
 600                 FindOccurrencesConfigurator config = new FindOccurrencesConfigurator();
 601                 config.setSignificantIdentifier(unitId);
 602                 List<String> propertyPaths = new ArrayList<>();
 603                 propertyPaths.add("derivedFrom.*");
 604                 config.setPropertyPaths(propertyPaths);
 605                 commitTransaction(state.getTx());
 606                 state.setTx(startTransaction());
 607                 try{
 608                         Pager<SpecimenOrObservationBase> existingSpecimens = cdmAppController.getOccurrenceService().findByTitle(config);
 609                         if(!existingSpecimens.getRecords().isEmpty()){
 610                             if(existingSpecimens.getRecords().size()==1){
 611                                 return existingSpecimens.getRecords().iterator().next();
 612                             }
 613                         }
 614
 615                 }catch(NullPointerException e){
 616                         logger.error("searching for existing specimen creates NPE: " + config.getSignificantIdentifier());
 617                         e.printStackTrace();
 618                 }
 619
 620
 621                 return null;
 622             }
 623
 624             protected abstract void importAssociatedUnits(STATE state, Object item, DerivedUnitFacade derivedUnitFacade);
 625
 626             /**
 627              * getFacade : get the DerivedUnitFacade based on the recordBasis
 628              * @param state
 629              *
 630              * @return DerivedUnitFacade
 631              */
 632             protected DerivedUnitFacade getFacade(STATE state) {
 633                 if (logger.isDebugEnabled()){
 634                     logger.info("getFacade()");
 635                 }
 636                 SpecimenOrObservationType type = null;
 637
 638                 // create specimen
 639                 if (NB((state.getDataHolder().getRecordBasis())) != null) {
 640                     if (state.getDataHolder().getRecordBasis().toLowerCase().startsWith("s") || state.getDataHolder().getRecordBasis().toLowerCase().indexOf("specimen")>-1) {// specimen
 641                         type = SpecimenOrObservationType.PreservedSpecimen;
 642                     }
 643                     if (state.getDataHolder().getRecordBasis().toLowerCase().startsWith("o") ||state.getDataHolder().getRecordBasis().toLowerCase().indexOf("observation")>-1 ) {
 644                         type = SpecimenOrObservationType.Observation;
 645                     }
 646                     if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("fossil")>-1){
 647                         type = SpecimenOrObservationType.Fossil;
 648                     }
 649                     if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("living")>-1) {
 650                         type = SpecimenOrObservationType.LivingSpecimen;
 651                     }
 652                     if (type == null) {
 653                         logger.info("The basis of record does not seem to be known: " + state.getDataHolder().getRecordBasis());
 654                         type = SpecimenOrObservationType.DerivedUnit;
 655                     }
 656                     // TODO fossils?
 657                 } else {
 658                     logger.info("The basis of record is null");
 659                     type = SpecimenOrObservationType.DerivedUnit;
 660                 }
 661                 DerivedUnitFacade derivedUnitFacade = DerivedUnitFacade.NewInstance(type);
 662                 return derivedUnitFacade;
 663             }
 664
 665             /**
 666              * Look if the Institution does already exist
 667              * @param institutionCode: a string with the institutioncode
 668              * @param config : the configurator
 669              * @return the Institution (existing or new)
 670              */
 671             protected Institution getInstitution(String institutionCode, STATE state) {
 672                 SpecimenImportConfiguratorBase config = state.getConfig();
 673                 Institution institution=null;
 674                 institution = (Institution)state.institutions.get(institutionCode);
 675                 if (institution != null){
 676                     return institution;
 677                 }
 678                 List<Institution> institutions;
 679                 try {
 680                     institutions = getAgentService().searchInstitutionByCode(institutionCode);
 681
 682                 } catch (Exception e) {
 683                     institutions = new ArrayList<Institution>();
 684                     logger.warn(e);
 685                 }
 686                 if (institutions.size() > 0 && config.isReuseExistingMetaData()) {
 687                     for (Institution institut:institutions){
 688                         try{
 689                             if (institut.getCode().equalsIgnoreCase(institutionCode)) {
 690                                 institution=institut;
 691                                 break;
 692                             }
 693                         }catch(Exception e){logger.warn("no institution code in the db");}
 694                     }
 695                 }
 696                 if (logger.isDebugEnabled()){
 697                     if(institution !=null) {
 698                         logger.info("getinstitution " + institution.toString());
 699                     }
 700                 }
 701                 if (institution == null){
 702                     // create institution
 703                     institution = Institution.NewInstance();
 704                     institution.setCode(institutionCode);
 705                     institution.setTitleCache(institutionCode, true);
 706                     UUID uuid = save(institution, state);
 707                 }
 708
 709
 710                 state.institutions.put(institutionCode, institution);
 711                 return institution;
 712             }
 713
 714             /**
 715              * Look if the Collection does already exist
 716              * @param collectionCode
 717              * @param collectionCode: a string
 718              * @param config : the configurator
 719              * @return the Collection (existing or new)
 720              */
 721             protected Collection getCollection(Institution institution, String collectionCode, STATE state) {
 722                 SpecimenImportConfiguratorBase config = state.getConfig();
 723                 Collection collection = null;
 724                 List<Collection> collections;
 725                 collection = (Collection) state.collections.get(collectionCode);
 726                 if (collection != null){
 727                     return collection;
 728                 }
 729                 try {
 730                     collections = getCollectionService().searchByCode(collectionCode);
 731                 } catch (Exception e) {
 732                     collections = new ArrayList<Collection>();
 733                 }
 734                 if (collections.size() > 0 && config.isReuseExistingMetaData()) {
 735                     for (Collection coll:collections){
 736                         if (coll.getCode() != null && coll.getInstitute() != null
 737                                 && coll.getCode().equalsIgnoreCase(collectionCode) && coll.getInstitute().equals(institution)) {
 738                             collection = coll;
 739                             break;
 740                         }
 741                     }
 742                 }
 743
 744                 if(collection == null){
 745                     collection =Collection.NewInstance();
 746                     collection.setCode(collectionCode);
 747                     collection.setInstitute(institution);
 748                     collection.setTitleCache(collectionCode);
 749                     UUID uuid = save(collection, state);
 750                 }
 751
 752
 753
 754                 state.collections.put(collectionCode, collection);
 755
 756                 return collection;
 757             }
 758
 759             /**
 760              * @param reference
 761              * @param citationDetail
 762              * @return
 763              */
 764             //FIXME this method is highly critical, because
 765             //  * it will have serious performance and memory problems with large databases
 766             //        (databases may easily have >1 Mio source records)
 767             //  * it does not make sense to search for existing sources and then clone them
 768             //    we need to search for existing references instead and use them (if exist)
 769             //    for our new source.
 770             protected IdentifiableSource getIdentifiableSource(Reference reference, String citationDetail) {
 771
 772                 IdentifiableSource sour = IdentifiableSource.NewInstance(OriginalSourceType.Import,null,null, reference,citationDetail);
 773                 return sour;
 774             }
 775
 776             /**
 777              * Add the hierarchy for a Taxon(add higher taxa)
 778              * @param classification
 779              * @param taxon: a taxon to add as a node
 780              * @param state: the ABCD import state
 781              */
 782             protected void addParentTaxon(Taxon taxon, STATE state, boolean preferredFlag, Classification classification){
 783                 INonViralName  nvname = taxon.getName();
 784                 Rank rank = nvname.getRank();
 785                 Taxon genus =null;
 786                 Taxon subgenus =null;
 787                 Taxon species = null;
 788                 Taxon subspecies = null;
 789                 Taxon parent = null;
 790                 if(rank!=null){
 791                     if (rank.isLower(Rank.GENUS() )){
 792                         String genusOrUninomial = nvname.getGenusOrUninomial();
 793                         TaxonName taxonName = getOrCreateTaxonName(genusOrUninomial, Rank.GENUS(), preferredFlag, state, -1);
 794                         genus = getOrCreateTaxonForName(taxonName, state);
 795                         if (genus == null){
 796                             logger.debug("The genus should not be null " + taxonName);
 797                         }
 798                         if (preferredFlag) {
 799                             parent = linkParentChildNode(null, genus, classification, state);
 800                         }
 801
 802                     }
 803                     if (rank.isLower(Rank.SUBGENUS())){
 804                         String prefix = nvname.getGenusOrUninomial();
 805                         String name = nvname.getInfraGenericEpithet();
 806                         if (name != null){
 807                             TaxonName taxonName = getOrCreateTaxonName(prefix+" "+name, Rank.SUBGENUS(), preferredFlag, state, -1);
 808                             subgenus = getOrCreateTaxonForName(taxonName, state);
 809                             if (preferredFlag) {
 810                                 parent = linkParentChildNode(genus, subgenus, classification, state);
 811                             }            }
 812                     }
 813                     if (rank.isLower(Rank.SPECIES())){
 814                         if (subgenus!=null){
 815                             String prefix = nvname.getGenusOrUninomial();
 816                             String name = nvname.getInfraGenericEpithet();
 817                             String spe = nvname.getSpecificEpithet();
 818                             if (spe != null){
 819                                 TaxonName taxonName = getOrCreateTaxonName(prefix+" "+name+" "+spe, Rank.SPECIES(), preferredFlag, state, -1);
 820                                 species = getOrCreateTaxonForName(taxonName, state);
 821                                 if (preferredFlag) {
 822                                     parent = linkParentChildNode(subgenus, species, classification, state);
 823                                 }
 824                             }
 825                         }
 826                         else{
 827                             String prefix = nvname.getGenusOrUninomial();
 828                             String name = nvname.getSpecificEpithet();
 829                             if (name != null){
 830                                 TaxonName taxonName = getOrCreateTaxonName(prefix+" "+name, Rank.SPECIES(), preferredFlag, state, -1);
 831                                 species = getOrCreateTaxonForName(taxonName, state);
 832                                 if (preferredFlag) {
 833                                     parent = linkParentChildNode(genus, species, classification, state);
 834                                 }
 835                             }
 836                         }
 837                     }
 838                     if (rank.isLower(Rank.INFRASPECIES())){
 839                         TaxonName taxonName = getOrCreateTaxonName(nvname.getFullTitleCache(), Rank.SUBSPECIES(), preferredFlag, state, -1);
 840                         subspecies = getOrCreateTaxonForName(taxonName, state);
 841                         if (preferredFlag) {
 842                             parent = linkParentChildNode(species, subspecies, classification, state);
 843                         }
 844                     }
 845                 }else{
 846                     //handle cf. and aff. taxa
 847                     String genusEpithet = null;
 848                     if (nvname.getTitleCache().contains("cf.")){
 849                         genusEpithet = nvname.getTitleCache().substring(0, nvname.getTitleCache().indexOf("cf."));
 850                     } else if (nvname.getTitleCache().contains("aff.")){
 851                         genusEpithet = nvname.getTitleCache().substring(0, nvname.getTitleCache().indexOf("aff."));
 852                     }
 853                     if (genusEpithet != null){
 854                     genusEpithet = genusEpithet.trim();
 855                     TaxonName taxonName = null;
 856                     if (genusEpithet.contains(" ")){
 857                         taxonName = getOrCreateTaxonName(genusEpithet, Rank.SPECIES(), preferredFlag, state, -1);
 858                     }else{
 859                         taxonName = getOrCreateTaxonName(genusEpithet, Rank.GENUS(), preferredFlag, state, -1);
 860                     }
 861                     genus = getOrCreateTaxonForName(taxonName, state);
 862                     if (genus == null){
 863                         logger.debug("The genus should not be null " + taxonName);
 864                     }
 865                     if (preferredFlag) {
 866                         parent = linkParentChildNode(null, genus, classification, state);
 867                     }
 868                     }
 869                 }
 870                 if (preferredFlag && parent!=taxon ) {
 871                     linkParentChildNode(parent, taxon, classification, state);
 872                 }
 873             }
 874
 875             /**
 876              * Link a parent to a child and save it in the current classification
 877              * @param parent: the higher Taxon
 878              * @param child : the lower (or current) Taxon
 879              * return the Taxon from the new created Node
 880              * @param classification
 881              * @param state
 882              */
 883             protected Taxon linkParentChildNode(Taxon parent, Taxon child, Classification classification, STATE state) {
 884                 TaxonNode node =null;
 885                 List<String> propertyPaths = new ArrayList<>();
 886             propertyPaths.add("childNodes");
 887                 if (parent != null) {
 888
 889                     parent = (Taxon) getTaxonService().load(parent.getUuid(), propertyPaths);
 890                     child = (Taxon) getTaxonService().load(child.getUuid(), propertyPaths);
 891                     //here we do not have to check if the taxon nodes already exists
 892                     //this is done by classification.addParentChild()
 893                     //do not add child node if it already exists
 894                     if(hasTaxonNodeInClassification(child, classification)){
 895                         return child;
 896                     }
 897                     else{
 898                         node = classification.addParentChild(parent, child, state.getRef(), "");
 899                         save(node, state);
 900                     }
 901                 }
 902                 else {
 903                     if (child == null){
 904                         logger.debug("The child should not be null!");
 905                     }
 906                     child = (Taxon) getTaxonService().find(child.getUuid());
 907                     //do not add child node if it already exists
 908                     if(hasTaxonNodeInClassification(child, classification)){
 909                         return child;
 910                     }
 911                     else{
 912                         node = classification.addChildTaxon(child, state.getRef(), null);
 913                         save(node, state);
 914                     }
 915                 }
 916                 if(node!=null){
 917                     state.getReport().addTaxonNode(node);
 918                     return node.getTaxon();
 919                 }
 920                 String message = "Could not create taxon node for " +child;
 921                 state.getReport().addInfoMessage(message);
 922                 logger.warn(message);
 923                 return null;
 924             }
 925
 926             protected Taxon getOrCreateTaxonForName(TaxonName taxonName, STATE state){
 927                 if (taxonName != null){
 928                 Set<Taxon> acceptedTaxa = taxonName.getTaxa();
 929                 if(acceptedTaxa.size()>0){
 930                     Taxon firstAcceptedTaxon = acceptedTaxa.iterator().next();
 931                     if(acceptedTaxa.size()>1){
 932                         String message = "More than one accepted taxon was found for taxon name: "
 933                                 + taxonName.getTitleCache() + "!\n" + firstAcceptedTaxon + "was chosen for "+state.getDerivedUnitBase();
 934                         state.getReport().addInfoMessage(message);
 935                         logger.warn(message);
 936                     }
 937                     else{
 938                         return firstAcceptedTaxon;
 939                     }
 940                 }
 941                 else{
 942                     Set<TaxonBase> taxonAndSynonyms = taxonName.getTaxonBases();
 943                     for (TaxonBase taxonBase : taxonAndSynonyms) {
 944                         if(taxonBase.isInstanceOf(Synonym.class)){
 945                             Synonym synonym = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
 946                             Taxon acceptedTaxonOfSynonym = synonym.getAcceptedTaxon();
 947                             if(acceptedTaxonOfSynonym == null){
 948                                 String message = "No accepted taxon could be found for taxon name: "
 949                                         + taxonName.getTitleCache()
 950                                         + "!";
 951                                 state.getReport().addInfoMessage(message);
 952                                 logger.warn(message);
 953                             }
 954                             else{
 955                                 return acceptedTaxonOfSynonym;
 956                             }
 957                         }
 958                     }
 959                 }
 960                 Taxon taxon = Taxon.NewInstance(taxonName, state.getRef());
 961                 save(taxon, state);
 962                 state.getReport().addTaxon(taxon);
 963                 logger.info("Created new taxon "+ taxon);
 964                 return taxon;
 965                 }
 966                 return null;
 967
 968             }
 969
 970             private boolean hasTaxonNodeInClassification(Taxon taxon, Classification classification){
 971                 if(taxon.getTaxonNodes()!=null){
 972                     for (TaxonNode node : taxon.getTaxonNodes()){
 973                         if(node.getClassification().equals(classification)){
 974                             return true;
 975                         }
 976                     }
 977                 }
 978                 return false;
 979             }
 980
 981             /**
 982              * HandleIdentifications : get the scientific names present in the ABCD
 983              * document and store link them with the observation/specimen data
 984              * @param state: the current ABCD import state
 985              * @param derivedUnitFacade : the current derivedunitfacade
 986              */
 987             protected void handleIdentifications(STATE state, DerivedUnitFacade derivedUnitFacade) {
 988                SpecimenImportConfiguratorBase config = state.getConfig();
 989
 990
 991                 String scientificName = "";
 992                 boolean preferredFlag = false;
 993
 994                 if (state.getDataHolder().getNomenclatureCode() == ""){
 995                     if (config.getNomenclaturalCode() != null){
 996                         if (config.getNomenclaturalCode() != null){
 997                             state.getDataHolder().setNomenclatureCode(config.getNomenclaturalCode().toString());
 998
 999                         }
1000                     }
1001                 }
1002
1003                 for (int i = 0; i < state.getDataHolder().getIdentificationList().size(); i++) {
1004                     Identification identification = state.getDataHolder().getIdentificationList().get(i);
1005                     scientificName = identification.getScientificName().replaceAll(" et ", " & ");
1006
1007                     String preferred = identification.getPreferred();
1008                     preferredFlag = false;
1009                     if (preferred != null || state.getDataHolder().getIdentificationList().size()==1){
1010                         if (state.getDataHolder().getIdentificationList().size()==1){
1011                             preferredFlag = true;
1012                         }else if (preferred != null && (preferred.equals("1") || preferred.toLowerCase().indexOf("true") != -1) ) {
1013                         preferredFlag = true;
1014                     }
1015
1016                     }
1017                     if (identification.getCode() != null){
1018                     if (identification.getCode().indexOf(':') != -1) {
1019                         state.getDataHolder().setNomenclatureCode(identification.getCode().split(COLON)[1]);
1020                     }
1021                     else{
1022                         state.getDataHolder().setNomenclatureCode(identification.getCode());
1023                     }
1024                     }
1025                     TaxonName taxonName = getOrCreateTaxonName(scientificName, null, preferredFlag, state, i);
1026                     Taxon taxon = getOrCreateTaxonForName(taxonName, state);
1027                     addTaxonNode(taxon, state,preferredFlag);
1028                     linkDeterminationEvent(state, taxon, preferredFlag, derivedUnitFacade, identification.getIdentifier(), identification.getDate(), identification.getModifier());
1029                 }
1030             }
1031
1032             /**
1033              * @param taxon : a taxon to add as a node
1034              * @param state : the ABCD import state
1035              */
1036             protected void addTaxonNode(Taxon taxon, STATE state, boolean preferredFlag) {
1037                 SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
1038                 logger.info("link taxon to a taxonNode "+taxon.getTitleCache());
1039                 //only add nodes if not already existing in current classification or default classification
1040
1041                 //check if node exists in current classification
1042                 //NOTE: we cannot use hasTaxonNodeInClassification() here because we are first creating it here
1043                 if (!existsInClassification(taxon,state.getClassification(), state)){
1044                     if(config.isMoveNewTaxaToDefaultClassification()){
1045                         //check if node exists in default classification
1046                         if (!existsInClassification(taxon, state.getDefaultClassification(true), state)){
1047                             addParentTaxon(taxon, state, preferredFlag, state.getDefaultClassification(true));
1048                         }
1049                     }else{
1050                         //add non-existing taxon to current classification
1051                         addParentTaxon(taxon, state, preferredFlag, state.getClassification());
1052                     }
1053
1054                 }
1055             }
1056
1057
1058             private boolean existsInClassification(Taxon taxon, Classification classification, STATE state){
1059                 boolean exist = false;
1060                 ICdmRepository cdmAppController = state.getConfig().getCdmAppController();
1061             if(cdmAppController==null){
1062                 cdmAppController = this;
1063             }
1064             if (classification != null){
1065                 if (!taxon.getTaxonNodes().isEmpty()){
1066                     for (TaxonNode node:taxon.getTaxonNodes()){
1067                         if (node.getClassification().equals(classification)){
1068                             return true;
1069                         }
1070                     }
1071                 }
1072 // we do not need this because we already searched for taxa in db in the previous steps
1073 //              List<UuidAndTitleCache<TaxonNode>> uuidAndTitleCacheOfAllTaxa = cdmAppController.getClassificationService().getTaxonNodeUuidAndTitleCacheOfAcceptedTaxaByClassification(classification.getUuid());
1074 //              if (uuidAndTitleCacheOfAllTaxa != null){
1075 //                      for (UuidAndTitleCache p : uuidAndTitleCacheOfAllTaxa){
1076 //                          try{
1077 //                              if(p.getTitleCache().equals(taxon.getTitleCache())) {
1078 //                                  exist = true;
1079 //                              }
1080 //                          }
1081 //                          catch(Exception e){
1082 //                              logger.warn("TaxonNode doesn't seem to have a taxon");
1083 //                          }
1084 //                      }
1085 //              }
1086             }
1087                 return exist;
1088             }
1089
1090             /**
1091              * join DeterminationEvent to the Taxon Object
1092              * @param state : the ABCD import state
1093              * @param taxon: the current Taxon
1094              * @param preferredFlag :if the current name is preferred
1095              * @param derivedFacade : the derived Unit Facade
1096              */
1097             @SuppressWarnings("rawtypes")
1098         protected void linkDeterminationEvent(STATE state, Taxon taxon, boolean preferredFlag,  DerivedUnitFacade derivedFacade, String identifierStr, String dateStr, String modifier) {
1099                 SpecimenImportConfiguratorBase config = state.getConfig();
1100                 if (logger.isDebugEnabled()){
1101                     logger.info("start linkdetermination with taxon:" + taxon.getUuid()+", "+taxon);
1102                 }
1103
1104                 DeterminationEvent determinationEvent = DeterminationEvent.NewInstance();
1105                 //determinationEvent.setTaxon(taxon);
1106                 determinationEvent.setTaxonName(taxon.getName());
1107                 determinationEvent.setPreferredFlag(preferredFlag);
1108
1109
1110                 determinationEvent.setIdentifiedUnit(state.getDerivedUnitBase());
1111                 if (state.getPersonStore().get(identifierStr) != null){
1112                     determinationEvent.setActor((AgentBase)state.getPersonStore().get(identifierStr));
1113                 } else if (identifierStr != null){
1114                     Person identifier = Person.NewTitledInstance(identifierStr);
1115                     determinationEvent.setActor(identifier);
1116                 }
1117                 if (dateStr != null){
1118                     determinationEvent.setTimeperiod(TimePeriodParser.parseString(dateStr));
1119                 }
1120                 if (modifier != null){
1121                     if (modifier.equals("cf.")){
1122                         determinationEvent.setModifier(DefinedTerm.DETERMINATION_MODIFIER_CONFER());
1123                     }else if (modifier.equals("aff.")){
1124                         determinationEvent.setModifier(DefinedTerm.DETERMINATION_MODIFIER_AFFINIS());
1125                     }
1126                 }
1127                 state.getDerivedUnitBase().addDetermination(determinationEvent);
1128
1129                 if (logger.isDebugEnabled()){
1130                     logger.debug("NB TYPES INFO: "+ state.getDataHolder().getStatusList().size());
1131                 }
1132                 for (SpecimenTypeDesignationStatus specimenTypeDesignationstatus : state.getDataHolder().getStatusList()) {
1133                     if (specimenTypeDesignationstatus != null) {
1134                         if (logger.isDebugEnabled()){
1135                             logger.debug("specimenTypeDesignationstatus :"+ specimenTypeDesignationstatus);
1136                         }
1137
1138                         ICdmRepository cdmAppController = config.getCdmAppController();
1139                         if(cdmAppController == null){
1140                             cdmAppController = this;
1141                         }
1142                         specimenTypeDesignationstatus = HibernateProxyHelper.deproxy(cdmAppController.getTermService().find(specimenTypeDesignationstatus.getUuid()), SpecimenTypeDesignationStatus.class);
1143                         //Designation
1144                         TaxonName name = taxon.getName();
1145                         SpecimenTypeDesignation designation = SpecimenTypeDesignation.NewInstance();
1146
1147                         designation.setTypeStatus(specimenTypeDesignationstatus);
1148                         designation.setTypeSpecimen(state.getDerivedUnitBase());
1149                         name.addTypeDesignation(designation, true);
1150                     }
1151                 }
1152                 save(state.getDerivedUnitBase(), state);
1153
1154                 for (String[] fullReference : state.getDataHolder().getReferenceList()) {
1155
1156
1157                     String strReference=fullReference[0];
1158                     String citationDetail = fullReference[1];
1159                     String citationURL = fullReference[2];
1160                     List<Reference> references = getReferenceService().listByTitleWithRestrictions(Reference.class, "strReference", MatchMode.EXACT, null, null, null, null, null);
1161
1162                     if (!references.isEmpty()){
1163                         Reference reference = null;
1164                         for (Reference refe: references) {
1165                             if (refe.getTitleCache().equalsIgnoreCase(strReference)) {
1166                                 reference =refe;
1167                                 break;
1168                             }
1169                         }
1170                         if (reference ==null){
1171                             reference = ReferenceFactory.newGeneric();
1172                             reference.setTitleCache(strReference, true);
1173                             save(reference, state);
1174                         }
1175                         determinationEvent.addReference(reference);
1176                     }
1177                 }
1178                 save(state.getDerivedUnitBase(), state);
1179
1180                 if (config.isAddIndividualsAssociationsSuchAsSpecimenAndObservations() && preferredFlag) {
1181                     //do not add IndividualsAssociation to non-preferred taxa
1182                     if (logger.isDebugEnabled()){
1183                         logger.debug("isDoCreateIndividualsAssociations");
1184                     }
1185
1186                     makeIndividualsAssociation(state, taxon, determinationEvent);
1187
1188                     save(state.getDerivedUnitBase(), state);
1189                 }
1190             }
1191
1192             /**
1193              * create and link each association (specimen, observation..) to the accepted taxon
1194              * @param state : the ABCD import state
1195              * @param taxon: the current Taxon
1196              * @param determinationEvent:the determinationevent
1197              */
1198             protected void makeIndividualsAssociation(STATE state, Taxon taxon, DeterminationEvent determinationEvent) {
1199                 SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
1200                 SpecimenUserInteraction sui = config.getSpecimenUserInteraction();
1201
1202                 if (logger.isDebugEnabled()){
1203                     logger.info("MAKE INDIVIDUALS ASSOCIATION");
1204                 }
1205
1206                 TaxonDescription taxonDescription = null;
1207                 Set<TaxonDescription> descriptions= taxon.getDescriptions();
1208                 if (state.getIndividualsAssociationDescriptionPerTaxon(taxon.getUuid()) != null){
1209                     taxonDescription = state.getIndividualsAssociationDescriptionPerTaxon(taxon.getUuid());
1210                 }
1211                if (taxonDescription == null && !descriptions.isEmpty() && state.getConfig().isReuseExistingDescriptiveGroups()){
1212                    for (TaxonDescription desc: descriptions){
1213                        if (desc.getTypes().contains(DescriptionType.INDIVIDUALS_ASSOCIATION)){
1214                            taxonDescription = desc;
1215                            break;
1216                        }
1217                    }
1218                }
1219
1220                if (taxonDescription == null){
1221                     taxonDescription = TaxonDescription.NewInstance(taxon, false);
1222                     taxonDescription.setTypes(EnumSet.of(DescriptionType.INDIVIDUALS_ASSOCIATION));
1223                     if(sourceNotLinkedToElement(taxonDescription,state.getRef(),null)) {
1224                         taxonDescription.addSource(OriginalSourceType.Import, null, null, state.getRef(), null);
1225                     }
1226                     state.setIndividualsAssociationDescriptionPerTaxon(taxonDescription);
1227                     taxon.addDescription(taxonDescription);
1228                 }
1229
1230                 //PREPARE REFERENCE QUESTIONS
1231
1232                 Map<String,OriginalSourceBase> sourceMap = new HashMap<>();
1233
1234                 List<IdentifiableSource> issTmp = new ArrayList<>();//getCommonService().list(IdentifiableSource.class, null, null, null, null);
1235                 List<DescriptionElementSource> issTmp2 = new ArrayList<>();//getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1236
1237                 Set<OriginalSourceBase> osbSet = new HashSet<>();
1238                 if(issTmp2!=null) {
1239                     osbSet.addAll(issTmp2);
1240                 }
1241                 if(issTmp!=null) {
1242                     osbSet.addAll(issTmp);
1243                 }
1244
1245
1246                 addToSourceMap(sourceMap, osbSet);
1247
1248
1249             if(sourceNotLinkedToElement(taxonDescription,state.getRef(),null)) {
1250                 taxonDescription.addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1251             }
1252
1253                 state.setIndividualsAssociationDescriptionPerTaxon(taxonDescription);
1254
1255                 IndividualsAssociation indAssociation = IndividualsAssociation.NewInstance();
1256                 Feature feature = makeFeature(state.getDerivedUnitBase());
1257                 indAssociation.setAssociatedSpecimenOrObservation(state.getDerivedUnitBase());
1258                 indAssociation.setFeature(feature);
1259
1260             if(sourceNotLinkedToElement(indAssociation,state.getImportReference(state.getActualAccessPoint()),null)) {
1261                 indAssociation.addSource(OriginalSourceType.Import,null, null, state.getImportReference(state.getActualAccessPoint()), null);
1262             }
1263             if(sourceNotLinkedToElement(state.getDerivedUnitBase(), state.getImportReference(state.getActualAccessPoint()),null)) {
1264                 state.getDerivedUnitBase().addSource(OriginalSourceType.Import,null, null, state.getImportReference(state.getActualAccessPoint()), null);
1265             }
1266             for (Reference citation : determinationEvent.getReferences()) {
1267                 if(sourceNotLinkedToElement(indAssociation,citation,null))
1268                 {
1269                     indAssociation.addSource(DescriptionElementSource.NewInstance(OriginalSourceType.Import, null, null, citation, null));
1270                 }
1271                 if(sourceNotLinkedToElement(state.getDerivedUnitBase(), state.getImportReference(state.getActualAccessPoint()),null)) {
1272                     state.getDerivedUnitBase().addSource(OriginalSourceType.Import,null, null, state.getImportReference(state.getActualAccessPoint()), null);
1273                 }
1274             }
1275
1276
1277                 taxonDescription.addElement(indAssociation);
1278
1279                 save(taxonDescription, state);
1280                 save(taxon, state);
1281                 state.getReport().addDerivate(state.getDerivedUnitBase(), config);
1282                 state.getReport().addIndividualAssociation(taxon, state.getDataHolder().getUnitID(), state.getDerivedUnitBase());
1283             }
1284
1285             /**
1286              * @param derivedUnitBase2
1287              * @param ref2
1288              * @param object
1289              * @return
1290              */
1291             private boolean sourceNotLinkedToElement(DerivedUnit derivedUnitBase2, Reference b, String d) {
1292                 Set<IdentifiableSource> linkedSources = derivedUnitBase2.getSources();
1293                 for (IdentifiableSource is:linkedSources){
1294                     Reference a = is.getCitation();
1295                     String c = is.getCitationMicroReference();
1296
1297                     boolean refMatch=false;
1298                     boolean microMatch=false;
1299
1300                     try{
1301                         if (a==null && b==null) {
1302                             refMatch=true;
1303                         }
1304                         if (a!=null && b!=null) {
1305                             if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1306                                 refMatch=true;
1307                             }
1308                         }
1309                     }catch(Exception e){}
1310
1311
1312                     try{
1313                         if (c==null && d==null) {
1314                             microMatch=true;
1315                         }
1316                         if(c!=null && d!=null) {
1317                             if(c.equalsIgnoreCase(d)) {
1318                                 microMatch=true;
1319                             }
1320                         }
1321                     }
1322                     catch(Exception e){}
1323
1324                     if (microMatch && refMatch) {
1325                         return false;
1326                     }
1327
1328
1329                 }
1330                 return true;
1331             }
1332
1333             private <T extends OriginalSourceBase> boolean  sourceNotLinkedToElement(ISourceable<T> sourcable, Reference reference, String microReference) {
1334                 Set<T> linkedSources = sourcable.getSources();
1335                 for (T is:linkedSources){
1336                     Reference unitReference = is.getCitation();
1337                     String unitMicroReference = is.getCitationMicroReference();
1338
1339                     boolean refMatch=false;
1340                     boolean microMatch=false;
1341
1342                     try{
1343                         if (unitReference==null && reference==null) {
1344                             refMatch=true;
1345                         }
1346                         if (unitReference!=null && reference!=null) {
1347                             if (unitReference.getTitleCache().equalsIgnoreCase(reference.getTitleCache())) {
1348                                 refMatch=true;
1349                             }
1350                         }
1351                     }catch(Exception e){}
1352
1353                     try{
1354                         if (unitMicroReference==null && microReference==null) {
1355                             microMatch=true;
1356                         }
1357                         if(unitMicroReference!=null && microReference!=null) {
1358                             if(unitMicroReference.equalsIgnoreCase(microReference)) {
1359                                 microMatch=true;
1360                             }
1361                         }
1362                     }
1363                     catch(Exception e){}
1364
1365                     if (microMatch && refMatch) {
1366                         return false;
1367                     }
1368                 }
1369                 return true;
1370             }
1371
1372             /**
1373              * look for the Feature object (FieldObs, Specimen,...)
1374              * @param unit : a specimen or obersvation base
1375              * @return the corresponding Feature
1376              */
1377             private Feature makeFeature(SpecimenOrObservationBase<?> unit) {
1378                 SpecimenOrObservationType type = unit.getRecordBasis();
1379
1380
1381
1382                 if (type.isFeatureObservation()){
1383                     return Feature.OBSERVATION();
1384                 }else if (type.isFeatureSpecimen()){
1385                     return Feature.SPECIMEN();
1386                 }else if (type == SpecimenOrObservationType.DerivedUnit){
1387                     return Feature.OBSERVATION();
1388                     //            return getFeature("Specimen or observation");
1389                 }else{
1390                     String message = "Unhandled record basis '%s' for defining individuals association feature type. Use default.";
1391                     logger.warn(String.format(message, type.getLabel()));
1392                     return Feature.OBSERVATION();
1393                     //            return getFeature("Specimen or observation");
1394
1395                 }
1396             }
1397
1398
1399             /**
1400              * @param sourceMap
1401              * @param osbSet
1402              */
1403             protected void addToSourceMap(Map<String, OriginalSourceBase> sourceMap, Set<OriginalSourceBase> osbSet) {
1404                 for( OriginalSourceBase osb:osbSet) {
1405                     if(osb.getCitation()!=null && osb.getCitationMicroReference() !=null  && !osb.getCitationMicroReference().isEmpty()) {
1406                         try{
1407                             sourceMap.put(osb.getCitation().getTitleCache()+ "---"+osb.getCitationMicroReference(),osb);
1408                         }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1409                     } else if(osb.getCitation()!=null){
1410                         try{
1411                             sourceMap.put(osb.getCitation().getTitleCache(),osb);
1412                         }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1413                     }
1414                 }
1415             }
1416
1417
1418
1419
1420 }