cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/SpecimenImportBase.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.specimen;
  11
  12
  13 import java.util.ArrayList;
  14 import java.util.HashMap;
  15 import java.util.HashSet;
  16 import java.util.List;
  17 import java.util.Map;
  18 import java.util.Set;
  19 import java.util.UUID;
  20
  21 import org.apache.log4j.Logger;
  22
  23 import eu.etaxonomy.cdm.api.application.ICdmRepository;
  24 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
  25 import eu.etaxonomy.cdm.api.service.config.FindOccurrencesConfigurator;
  26 import eu.etaxonomy.cdm.api.service.pager.Pager;
  27 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  28 import eu.etaxonomy.cdm.io.common.CdmImportBase;
  29 import eu.etaxonomy.cdm.io.common.IImportConfigurator;
  30 import eu.etaxonomy.cdm.io.specimen.abcd206.in.Identification;
  31 import eu.etaxonomy.cdm.io.specimen.abcd206.in.SpecimenImportReport;
  32 import eu.etaxonomy.cdm.model.agent.AgentBase;
  33 import eu.etaxonomy.cdm.model.agent.Institution;
  34 import eu.etaxonomy.cdm.model.agent.Person;
  35 import eu.etaxonomy.cdm.model.agent.Team;
  36 import eu.etaxonomy.cdm.model.common.CdmBase;
  37 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  38 import eu.etaxonomy.cdm.model.common.LanguageString;
  39 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  40 import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
  41 import eu.etaxonomy.cdm.model.description.Feature;
  42 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  43 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  44 import eu.etaxonomy.cdm.model.name.INonViralName;
  45 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  46 import eu.etaxonomy.cdm.model.name.Rank;
  47 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
  48 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
  49 import eu.etaxonomy.cdm.model.name.TaxonName;
  50 import eu.etaxonomy.cdm.model.name.TaxonNameFactory;
  51 import eu.etaxonomy.cdm.model.occurrence.Collection;
  52 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  53 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
  54 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
  55 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationType;
  56 import eu.etaxonomy.cdm.model.reference.ISourceable;
  57 import eu.etaxonomy.cdm.model.reference.OriginalSourceBase;
  58 import eu.etaxonomy.cdm.model.reference.OriginalSourceType;
  59 import eu.etaxonomy.cdm.model.reference.Reference;
  60 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  61 import eu.etaxonomy.cdm.model.taxon.Classification;
  62 import eu.etaxonomy.cdm.model.taxon.Synonym;
  63 import eu.etaxonomy.cdm.model.taxon.Taxon;
  64 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  65 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  66 import eu.etaxonomy.cdm.model.term.DefinedTerm;
  67 import eu.etaxonomy.cdm.persistence.query.MatchMode;
  68 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
  69 import eu.etaxonomy.cdm.strategy.parser.ParserProblem;
  70 import eu.etaxonomy.cdm.strategy.parser.TimePeriodParser;
  71
  72
  73 /**
  74  * @author p.kelbert
  75  * @since 20.10.2008
  76  */
  77 public abstract class SpecimenImportBase<CONFIG extends IImportConfigurator, STATE extends SpecimenImportStateBase>
  78         extends CdmImportBase<CONFIG, STATE> {
  79
  80     private static final long serialVersionUID = 4423065367998125678L;
  81     private static final Logger logger = Logger.getLogger(SpecimenImportBase.class);
  82
  83         protected static final UUID SPECIMEN_SCAN_TERM = UUID.fromString("acda15be-c0e2-4ea8-8783-b9b0c4ad7f03");
  84
  85         private static final String COLON = ":";
  86
  87         protected Map<String, DefinedTerm> kindOfUnitsMap;
  88
  89
  90         @Override
  91     protected abstract void doInvoke(STATE state);
  92
  93         /**
  94          * Handle a single unit
  95          * @param state
  96          * @param item
  97          */
  98         protected abstract void handleSingleUnit(STATE state, Object item) ;
  99
 100
 101
 102         protected TaxonName getOrCreateTaxonName(String scientificName, Rank rank, boolean preferredFlag, STATE state, int unitIndexInAbcdFile){
 103             TaxonName taxonName = null;
 104         SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
 105
 106         //check atomised name data for rank
 107         //new name will be created
 108         TaxonName atomisedTaxonName = null;
 109         if (rank==null && unitIndexInAbcdFile>=0 && ((state.getDataHolder().getAtomisedIdentificationList() != null && !state.getDataHolder().getAtomisedIdentificationList().isEmpty())|| state.getDataHolder().getAtomisedIdentificationList().size() > 0)) {
 110             atomisedTaxonName = setTaxonNameByType(state.getDataHolder().getAtomisedIdentificationList().get(unitIndexInAbcdFile), scientificName, state);
 111             if(atomisedTaxonName!=null){
 112                 rank = atomisedTaxonName.getRank();
 113             }
 114         }
 115         if(config.isReuseExistingTaxaWhenPossible()){
 116             TaxonName parsedName = atomisedTaxonName;
 117             if(parsedName==null){
 118
 119                 parsedName = parseScientificName(scientificName, state, state.getReport(), rank);
 120
 121             }
 122             atomisedTaxonName = parsedName;
 123             if(config.isIgnoreAuthorship() && parsedName!=null){// && preferredFlag){
 124                 // do not ignore authorship for non-preferred names because they need
 125                 // to be created for the determination history
 126                 String nameCache = TaxonName.castAndDeproxy(parsedName).getNameCache();
 127                 List<TaxonName> names = getNameService().findNamesByNameCache(nameCache, MatchMode.EXACT, null);
 128                 if (!names.isEmpty()){
 129                      taxonName = getBestMatchingName(scientificName, new ArrayList<>(names), state);
 130                 }
 131                 if (taxonName == null && !names.isEmpty()){
 132                     taxonName = names.get(0);
 133                 }
 134
 135             } else {
 136                 //search for existing names
 137                 List<TaxonName> names = getNameService().listByTitleWithRestrictions(TaxonName.class, scientificName, MatchMode.EXACT, null, null, null, null, null);
 138                 taxonName = getBestMatchingName(scientificName, names, state);
 139                 //still nothing found -> try with the atomised name full title cache
 140                 if(taxonName==null && atomisedTaxonName!=null){
 141                     names = getNameService().listByTitleWithRestrictions(TaxonName.class, atomisedTaxonName.getFullTitleCache(), MatchMode.EXACT, null, null, null, null, null);
 142                     taxonName = getBestMatchingName(atomisedTaxonName.getTitleCache(), names, state);
 143                     //still nothing found -> try with the atomised name title cache
 144                     if(taxonName==null){
 145                         names = getNameService().listByTitleWithRestrictions(TaxonName.class, atomisedTaxonName.getTitleCache(), MatchMode.EXACT, null, null, null, null, null);
 146                         taxonName = getBestMatchingName(atomisedTaxonName.getTitleCache(), names, state);
 147                     }
 148                 }
 149
 150             }
 151
 152         }
 153
 154         if(taxonName==null && atomisedTaxonName!=null){
 155             taxonName = atomisedTaxonName;
 156             state.getReport().addName(taxonName);
 157             logger.info("Created new taxon name "+taxonName);
 158             if(taxonName.hasProblem()){
 159                 state.getReport().addInfoMessage(String.format("Created %s with parsing problems", taxonName));
 160             }
 161             if(!atomisedTaxonName.getTitleCache().equals(scientificName)){
 162                 state.getReport().addInfoMessage(String.format("Taxon %s was parsed as %s", scientificName, atomisedTaxonName.getTitleCache()));
 163             }
 164         }
 165         else if(taxonName==null){
 166             //create new taxon name
 167
 168             if (state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNAFP)){
 169                 taxonName = TaxonNameFactory.NewBotanicalInstance(rank);
 170             }else if (state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICZN)){
 171                 taxonName = TaxonNameFactory.NewZoologicalInstance(rank);
 172             }else{
 173                 taxonName = TaxonNameFactory.NewNonViralInstance(rank);
 174             }
 175             taxonName.setFullTitleCache(scientificName,true);
 176             taxonName.setTitleCache(scientificName, true);
 177             state.getReport().addName(taxonName);
 178             logger.info("Created new taxon name "+taxonName);
 179         }
 180         if (taxonName != null){
 181             state.names.put(taxonName.getNameCache(), taxonName);
 182         }
 183         if(!taxonName.isPersited()) {
 184             save(taxonName, state);
 185         }
 186         return taxonName;
 187     }
 188
 189          protected TaxonName getBestMatchingName(String scientificName, java.util.Collection<TaxonName> names, STATE state){
 190                 Set<TaxonName> namesWithAcceptedTaxa = new HashSet<>();
 191                 List<TaxonName> namesWithAcceptedTaxaInClassification = new ArrayList<>();
 192                 for (TaxonName name : names) {
 193                     if(!name.getTaxa().isEmpty()){
 194                         Set<Taxon> taxa = name.getTaxa();
 195                         for (Taxon taxon:taxa){
 196                             if (!taxon.getTaxonNodes().isEmpty()){
 197                                 //use only taxa included in a classification
 198                                 for (TaxonNode node:taxon.getTaxonNodes()){
 199                                     if (state.getClassification() != null && node.getClassification().equals(state.getClassification())){
 200                                         namesWithAcceptedTaxaInClassification.add(name);
 201                                     }else {
 202                                         namesWithAcceptedTaxa.add(name);
 203                                     }
 204                                 }
 205
 206                             }
 207                         }
 208
 209                     }
 210                 }
 211                 String message = String.format("More than one taxon name was found for %s, maybe in other classifications!", scientificName);
 212                 //check for names with accepted taxa in classification
 213                 if(namesWithAcceptedTaxaInClassification.size()>0){
 214                 if(namesWithAcceptedTaxaInClassification.size()>1){
 215
 216                     state.getReport().addInfoMessage(message);
 217                     logger.warn(message);
 218                     return null;
 219                 }
 220                 return namesWithAcceptedTaxaInClassification.iterator().next();
 221             }
 222               //check for any names with accepted taxa
 223                 if(namesWithAcceptedTaxa.size()>0){
 224                     if(namesWithAcceptedTaxa.size()>1){
 225
 226                         state.getReport().addInfoMessage(message);
 227                         logger.warn(message);
 228                         return null;
 229                     }
 230                     return namesWithAcceptedTaxa.iterator().next();
 231                 }
 232                 //no names with accepted taxa found -> check accepted taxa of synonyms
 233                 List<Taxon> taxaFromSynonyms = new ArrayList<>();
 234                 for (TaxonName name : names) {
 235                     Set<TaxonBase> taxonBases = name.getTaxonBases();
 236                     for (TaxonBase taxonBase : taxonBases) {
 237                         if(taxonBase.isInstanceOf(Synonym.class)){
 238                             Synonym synonym = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
 239                             taxaFromSynonyms.add(synonym.getAcceptedTaxon());
 240                         }
 241                     }
 242                 }
 243                 if(taxaFromSynonyms.size()>0){
 244                     if(taxaFromSynonyms.size()>1){
 245                         state.getReport().addInfoMessage(message);
 246                         logger.warn(message);
 247                         return null;
 248                     }
 249                     return taxaFromSynonyms.iterator().next().getName();
 250                 }
 251                 //no accepted and no synonyms -> return one of the names and create a new taxon
 252                 if (names.isEmpty()){
 253                     return null;
 254                 }else{
 255                     return names.iterator().next();
 256                 }
 257             }
 258          /**
 259              * Parse automatically the scientific name
 260              * @param scientificName the scientific name to parse
 261              * @param state the current import state
 262              * @param report the import report
 263              * @return a parsed name
 264              */
 265
 266             protected TaxonName parseScientificName(String scientificName, STATE state, SpecimenImportReport report, Rank rank) {
 267
 268                 NonViralNameParserImpl nvnpi = NonViralNameParserImpl.NewInstance();
 269                 TaxonName taxonName = null;
 270                 boolean problem = false;
 271
 272                 if (logger.isDebugEnabled()){
 273                     logger.debug("parseScientificName " + state.getDataHolder().getNomenclatureCode().toString());
 274                 }
 275
 276                 if (state.getDataHolder().getNomenclatureCode() != null && (state.getDataHolder().getNomenclatureCode().toString().equals("Zoological") || state.getDataHolder().getNomenclatureCode().toString().contains("ICZN"))) {
 277                     taxonName = (TaxonName)nvnpi.parseFullName(scientificName, NomenclaturalCode.ICZN, rank);
 278                     if (taxonName.hasProblem()) {
 279                         problem = true;
 280                     }
 281                 }
 282                 else if (state.getDataHolder().getNomenclatureCode() != null && (state.getDataHolder().getNomenclatureCode().toString().equals("Botanical") || state.getDataHolder().getNomenclatureCode().toString().contains("ICBN")  || state.getDataHolder().getNomenclatureCode().toString().contains("ICNAFP"))) {
 283                     taxonName = (TaxonName)nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNAFP, rank);
 284                     if (taxonName.hasProblem()) {
 285                         problem = true;
 286                     }
 287                 }
 288                 else if (state.getDataHolder().getNomenclatureCode() != null && (state.getDataHolder().getNomenclatureCode().toString().equals("Bacterial") || state.getDataHolder().getNomenclatureCode().toString().contains("ICBN"))) {
 289                     taxonName = (TaxonName)nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNB, rank);
 290                     if (taxonName.hasProblem()) {
 291                         problem = true;
 292                     }
 293                 }
 294                 else if (state.getDataHolder().getNomenclatureCode() != null && (state.getDataHolder().getNomenclatureCode().toString().equals("Cultivar") || state.getDataHolder().getNomenclatureCode().toString().contains("ICNCP"))) {
 295                     taxonName = (TaxonName)nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNCP, rank);
 296                     if (taxonName.hasProblem()) {
 297                         problem = true;
 298                     }
 299                 }
 300                 if (problem) {
 301                     String message = String.format("Parsing problems for %s", scientificName);
 302                     if(taxonName!=null){
 303                         for (ParserProblem parserProblem : taxonName.getParsingProblems()) {
 304                             message += "\n\t- "+parserProblem;
 305                         }
 306                     }
 307                     report.addInfoMessage(message);
 308                     logger.info(message);
 309                 }
 310                 return taxonName;
 311
 312             }
 313
 314             /**
 315              * Create the name without automatic parsing, either because it failed, or because the user deactivated it.
 316              * The name is built upon the ABCD fields
 317              * @param atomisedMap : the ABCD atomised fields
 318              * @param fullName : the full scientific name
 319              * @param state
 320              * @return the corresponding Botanical or Zoological or... name
 321              */
 322             protected TaxonName setTaxonNameByType(
 323                     HashMap<String, String> atomisedMap, String fullName, STATE state) {
 324                 boolean problem = false;
 325                 if (logger.isDebugEnabled()){
 326                     logger.debug("settaxonnamebytype " + state.getDataHolder().getNomenclatureCode().toString());
 327                 }
 328
 329                 if (state.getDataHolder().getNomenclatureCode().equals("Zoological") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICZN.getUuid())) {
 330                     TaxonName taxonName = TaxonNameFactory.NewZoologicalInstance(null);
 331                     taxonName.setFullTitleCache(fullName, true);
 332                     taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
 333                     taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
 334                     taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap,"SpeciesEpithet")));
 335                     taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap,"SubspeciesEpithet")));
 336
 337                     if (taxonName.getGenusOrUninomial() != null){
 338                         taxonName.setRank(Rank.GENUS());
 339                     }
 340
 341                     if (taxonName.getInfraGenericEpithet() != null){
 342                         taxonName.setRank(Rank.SUBGENUS());
 343                     }
 344
 345                     if (taxonName.getSpecificEpithet() != null){
 346                         taxonName.setRank(Rank.SPECIES());
 347                     }
 348
 349                     if (taxonName.getInfraSpecificEpithet() != null){
 350                         taxonName.setRank(Rank.SUBSPECIES());
 351                     }
 352
 353                     Team team = null;
 354                     if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 355                         team = Team.NewInstance();
 356                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
 357                     }
 358                     else {
 359                         if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 360                             team = Team.NewInstance();
 361                             team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
 362                         }
 363                     }
 364                     if (team != null) {
 365                         taxonName.setBasionymAuthorship(team);
 366                     }
 367                     else {
 368                         if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 369                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
 370                         }
 371                         else if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 372                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamAndYear"));
 373                         }
 374                     }
 375                     if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
 376                         team = Team.NewInstance();
 377                         team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
 378                         taxonName.setCombinationAuthorship(team);
 379                     }
 380                     if (taxonName.hasProblem()) {
 381                         logger.info("pb ICZN");
 382                         problem = true;
 383                     }
 384                     else {
 385                         return taxonName;
 386                     }
 387                 }
 388                 else if (state.getDataHolder().getNomenclatureCode().equals("Botanical") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNAFP.getUuid())) {
 389                     TaxonName taxonName = parseScientificName(fullName, state, state.getReport(), null);
 390                     if (taxonName != null){
 391                         return taxonName;
 392                     }
 393                     else{
 394                         taxonName = TaxonNameFactory.NewBotanicalInstance(null);
 395                     }
 396                     taxonName.setFullTitleCache(fullName, true);
 397                     taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
 398                     taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "FirstEpithet")));
 399                     taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "InfraSpeEpithet")));
 400                     try {
 401                         taxonName.setRank(Rank.getRankByName(getFromMap(atomisedMap, "Rank")));
 402                     } catch (Exception e) {
 403                         if (taxonName.getInfraSpecificEpithet() != null){
 404                             taxonName.setRank(Rank.SUBSPECIES());
 405                         }
 406                         else if (taxonName.getSpecificEpithet() != null){
 407                             taxonName.setRank(Rank.SPECIES());
 408                         }
 409                         else if (taxonName.getInfraGenericEpithet() != null){
 410                             taxonName.setRank(Rank.SUBGENUS());
 411                         }
 412                         else if (taxonName.getGenusOrUninomial() != null){
 413                             taxonName.setRank(Rank.GENUS());
 414                         }
 415                     }
 416                     Team team = null;
 417                     if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 418                         team = Team.NewInstance();
 419                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
 420                         taxonName.setBasionymAuthorship(team);
 421                     }
 422                     if (getFromMap(atomisedMap, "AuthorTeam") != null) {
 423                         team = Team.NewInstance();
 424                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeam"), true);
 425                         taxonName.setCombinationAuthorship(team);
 426                     }
 427                     if (team == null) {
 428                         if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 429                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
 430                         }
 431                         else if (getFromMap(atomisedMap, "AuthorTeam") != null) {
 432                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeam"));
 433                         }
 434                     }
 435                     if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
 436                         team = Team.NewInstance();
 437                         team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
 438                         taxonName.setCombinationAuthorship(team);
 439                     }
 440                     if (taxonName.hasProblem()) {
 441                         logger.info("pb ICBN");
 442                         problem = true;
 443                     }
 444                     else {
 445                         return taxonName;
 446                     }
 447                 }
 448                 else if (state.getDataHolder().getNomenclatureCode().equals("Bacterial") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNB.getUuid())) {
 449                     TaxonName taxonName = TaxonNameFactory.NewBacterialInstance(null);
 450                     taxonName.setFullTitleCache(fullName, true);
 451                     taxonName.setGenusOrUninomial(getFromMap(atomisedMap, "Genus"));
 452                     taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
 453                     taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "Species")));
 454                     taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "SubspeciesEpithet")));
 455
 456                     if (taxonName.getGenusOrUninomial() != null){
 457                         taxonName.setRank(Rank.GENUS());
 458                     }
 459                     else if (taxonName.getInfraGenericEpithet() != null){
 460                         taxonName.setRank(Rank.SUBGENUS());
 461                     }
 462                     else if (taxonName.getSpecificEpithet() != null){
 463                         taxonName.setRank(Rank.SPECIES());
 464                     }
 465                     else if (taxonName.getInfraSpecificEpithet() != null){
 466                         taxonName.setRank(Rank.SUBSPECIES());
 467                     }
 468
 469                     if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 470                         Team team = Team.NewInstance();
 471                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
 472                         taxonName.setCombinationAuthorship(team);
 473                     }
 474                     if (getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear") != null) {
 475                         Team team = Team.NewInstance();
 476                         team.setTitleCache(getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear"), true);
 477                         taxonName.setBasionymAuthorship(team);
 478                     }
 479                     if (taxonName.hasProblem()) {
 480                         logger.info("pb ICNB");
 481                         problem = true;
 482                     }
 483                     else {
 484                         return taxonName;
 485                     }
 486                 }
 487                 else if (state.getDataHolder().getNomenclatureCode().equals("Cultivar")) {
 488                     TaxonName taxonName = TaxonNameFactory.NewCultivarInstance(null);
 489
 490                     if (taxonName.hasProblem()) {
 491                         logger.info("pb ICNCP");
 492                         problem = true;
 493                     }
 494                     else {
 495                         return taxonName;
 496                     }
 497                     return taxonName;
 498                 }
 499
 500                 if (problem) {
 501                     logger.info("Problem im setTaxonNameByType ");
 502                     TaxonName taxonName = TaxonNameFactory.NewNonViralInstance(null);
 503                     taxonName.setFullTitleCache(fullName, true);
 504                     return taxonName;
 505                 }
 506                 TaxonName tn = TaxonNameFactory.NewNonViralInstance(null);
 507                 return tn;
 508             }
 509
 510             /**
 511              * Get a formated string from a hashmap
 512              * @param atomisedMap
 513              * @param key
 514              * @return
 515              */
 516             private String getFromMap(HashMap<String, String> atomisedMap, String key) {
 517                 String value = null;
 518                 if (atomisedMap.containsKey(key)) {
 519                     value = atomisedMap.get(key);
 520                 }
 521
 522                 try {
 523                     if (value != null && key.matches(".*Year.*")) {
 524                         value = value.trim();
 525                         if (value.matches("[a-z A-Z ]*[0-9]{4}$")) {
 526                             String tmp = value.split("[0-9]{4}$")[0];
 527                             int year = Integer.parseInt(value.split(tmp)[1]);
 528                             if (year >= 1752) {
 529                                 value = tmp;
 530                             }
 531                             else {
 532                                 value = null;
 533                             }
 534                         }
 535                         else {
 536                             value = null;
 537                         }
 538                     }
 539                 }
 540                 catch (Exception e) {
 541                     value = null;
 542                 }
 543                 return value;
 544             }
 545
 546             /**
 547              * Very fast and dirty implementation to allow handling of transient objects as described in
 548              * https://dev.e-taxonomy.eu/trac/ticket/3726
 549              *
 550              * Not yet complete.
 551              *
 552              * @param cdmBase
 553              * @param state
 554              */
 555             protected UUID save(CdmBase cdmBase, SpecimenImportStateBase state) {
 556                 ICdmRepository cdmRepository = state.getConfig().getCdmAppController();
 557                 if (cdmRepository == null){
 558                     cdmRepository = this;
 559                 }
 560
 561                 if (cdmBase.isInstanceOf(LanguageString.class)){
 562                     return cdmRepository.getTermService().saveLanguageData(CdmBase.deproxy(cdmBase, LanguageString.class));
 563                 }else if (cdmBase.isInstanceOf(SpecimenOrObservationBase.class)){
 564                     SpecimenOrObservationBase specimen = CdmBase.deproxy(cdmBase, SpecimenOrObservationBase.class);
 565
 566                     return cdmRepository.getOccurrenceService().saveOrUpdate(specimen);
 567                 }else if (cdmBase.isInstanceOf(Reference.class)){
 568                     return cdmRepository.getReferenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, Reference.class));
 569                 }else if (cdmBase.isInstanceOf(Classification.class)){
 570                     return cdmRepository.getClassificationService().saveOrUpdate(CdmBase.deproxy(cdmBase, Classification.class));
 571                 }else if (cdmBase.isInstanceOf(AgentBase.class)){
 572                     return cdmRepository.getAgentService().saveOrUpdate(CdmBase.deproxy(cdmBase, AgentBase.class));
 573                 }else if (cdmBase.isInstanceOf(Collection.class)){
 574                     return cdmRepository.getCollectionService().saveOrUpdate(CdmBase.deproxy(cdmBase, Collection.class));
 575                 }else if (cdmBase.isInstanceOf(DescriptionBase.class)){
 576                     DescriptionBase description = CdmBase.deproxy(cdmBase, DescriptionBase.class);
 577
 578                     return cdmRepository.getDescriptionService().saveOrUpdate(description);
 579                 }else if (cdmBase.isInstanceOf(TaxonBase.class)){
 580                     return cdmRepository.getTaxonService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonBase.class));
 581                 }else if (cdmBase.isInstanceOf(TaxonName.class)){
 582                     return cdmRepository.getNameService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonName.class));
 583                 }else if (cdmBase.isInstanceOf(TaxonNode.class)){
 584                 return cdmRepository.getTaxonNodeService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonNode.class));
 585             }else{
 586                     throw new IllegalArgumentException("Class not supported in save method: " + CdmBase.deproxy(cdmBase, CdmBase.class).getClass().getSimpleName());
 587                 }
 588
 589             }
 590
 591
 592             protected SpecimenOrObservationBase findExistingSpecimen(String unitId, SpecimenImportStateBase state){
 593                 ICdmRepository cdmAppController = state.getConfig().getCdmAppController();
 594                 if(cdmAppController==null){
 595                     cdmAppController = this;
 596                 }
 597                 FindOccurrencesConfigurator config = new FindOccurrencesConfigurator();
 598                 config.setSignificantIdentifier(unitId);
 599                 List<String> propertyPaths = new ArrayList<>();
 600                 propertyPaths.add("derivedFrom.*");
 601                 config.setPropertyPaths(propertyPaths);
 602                 commitTransaction(state.getTx());
 603                 state.setTx(startTransaction());
 604                 try{
 605                         Pager<SpecimenOrObservationBase> existingSpecimens = cdmAppController.getOccurrenceService().findByTitle(config);
 606                         if(!existingSpecimens.getRecords().isEmpty()){
 607                             if(existingSpecimens.getRecords().size()==1){
 608                                 return existingSpecimens.getRecords().iterator().next();
 609                             }
 610                         }
 611
 612                 }catch(NullPointerException e){
 613                         logger.error("searching for existing specimen creates NPE: " + config.getSignificantIdentifier());
 614                         e.printStackTrace();
 615                 }
 616
 617
 618                 return null;
 619             }
 620
 621             protected abstract void importAssociatedUnits(STATE state, Object item, DerivedUnitFacade derivedUnitFacade);
 622
 623             /**
 624              * getFacade : get the DerivedUnitFacade based on the recordBasis
 625              * @param state
 626              *
 627              * @return DerivedUnitFacade
 628              */
 629             protected DerivedUnitFacade getFacade(STATE state) {
 630                 if (logger.isDebugEnabled()){
 631                     logger.info("getFacade()");
 632                 }
 633                 SpecimenOrObservationType type = null;
 634
 635                 // create specimen
 636                 if (NB((state.getDataHolder().getRecordBasis())) != null) {
 637                     if (state.getDataHolder().getRecordBasis().toLowerCase().startsWith("s") || state.getDataHolder().getRecordBasis().toLowerCase().indexOf("specimen")>-1) {// specimen
 638                         type = SpecimenOrObservationType.PreservedSpecimen;
 639                     }
 640                     if (state.getDataHolder().getRecordBasis().toLowerCase().startsWith("o") ||state.getDataHolder().getRecordBasis().toLowerCase().indexOf("observation")>-1 ) {
 641                         type = SpecimenOrObservationType.Observation;
 642                     }
 643                     if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("fossil")>-1){
 644                         type = SpecimenOrObservationType.Fossil;
 645                     }
 646                     if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("living")>-1) {
 647                         type = SpecimenOrObservationType.LivingSpecimen;
 648                     }
 649                     if (type == null) {
 650                         logger.info("The basis of record does not seem to be known: " + state.getDataHolder().getRecordBasis());
 651                         type = SpecimenOrObservationType.DerivedUnit;
 652                     }
 653                     // TODO fossils?
 654                 } else {
 655                     logger.info("The basis of record is null");
 656                     type = SpecimenOrObservationType.DerivedUnit;
 657                 }
 658                 DerivedUnitFacade derivedUnitFacade = DerivedUnitFacade.NewInstance(type);
 659                 return derivedUnitFacade;
 660             }
 661
 662             /**
 663              * Look if the Institution does already exist
 664              * @param institutionCode: a string with the institutioncode
 665              * @param config : the configurator
 666              * @return the Institution (existing or new)
 667              */
 668             protected Institution getInstitution(String institutionCode, STATE state) {
 669                 SpecimenImportConfiguratorBase config = state.getConfig();
 670                 Institution institution=null;
 671                 institution = (Institution)state.institutions.get(institutionCode);
 672                 if (institution != null){
 673                     return institution;
 674                 }
 675                 List<Institution> institutions;
 676                 try {
 677                     institutions = getAgentService().searchInstitutionByCode(institutionCode);
 678
 679                 } catch (Exception e) {
 680                     institutions = new ArrayList<Institution>();
 681                     logger.warn(e);
 682                 }
 683                 if (institutions.size() > 0 && config.isReuseExistingMetaData()) {
 684                     for (Institution institut:institutions){
 685                         try{
 686                             if (institut.getCode().equalsIgnoreCase(institutionCode)) {
 687                                 institution=institut;
 688                                 break;
 689                             }
 690                         }catch(Exception e){logger.warn("no institution code in the db");}
 691                     }
 692                 }
 693                 if (logger.isDebugEnabled()){
 694                     if(institution !=null) {
 695                         logger.info("getinstitution " + institution.toString());
 696                     }
 697                 }
 698                 if (institution == null){
 699                     // create institution
 700                     institution = Institution.NewInstance();
 701                     institution.setCode(institutionCode);
 702                     institution.setTitleCache(institutionCode, true);
 703                     UUID uuid = save(institution, state);
 704                 }
 705
 706
 707                 state.institutions.put(institutionCode, institution);
 708                 return institution;
 709             }
 710
 711             /**
 712              * Look if the Collection does already exist
 713              * @param collectionCode
 714              * @param collectionCode: a string
 715              * @param config : the configurator
 716              * @return the Collection (existing or new)
 717              */
 718             protected Collection getCollection(Institution institution, String collectionCode, STATE state) {
 719                 SpecimenImportConfiguratorBase config = state.getConfig();
 720                 Collection collection = null;
 721                 List<Collection> collections;
 722                 collection = (Collection) state.collections.get(collectionCode);
 723                 if (collection != null){
 724                     return collection;
 725                 }
 726                 try {
 727                     collections = getCollectionService().searchByCode(collectionCode);
 728                 } catch (Exception e) {
 729                     collections = new ArrayList<Collection>();
 730                 }
 731                 if (collections.size() > 0 && config.isReuseExistingMetaData()) {
 732                     for (Collection coll:collections){
 733                         if (coll.getCode() != null && coll.getInstitute() != null
 734                                 && coll.getCode().equalsIgnoreCase(collectionCode) && coll.getInstitute().equals(institution)) {
 735                             collection = coll;
 736                             break;
 737                         }
 738                     }
 739                 }
 740
 741                 if(collection == null){
 742                     collection =Collection.NewInstance();
 743                     collection.setCode(collectionCode);
 744                     collection.setInstitute(institution);
 745                     collection.setTitleCache(collectionCode);
 746                     UUID uuid = save(collection, state);
 747                 }
 748
 749
 750
 751                 state.collections.put(collectionCode, collection);
 752
 753                 return collection;
 754             }
 755
 756             /**
 757              * @param reference
 758              * @param citationDetail
 759              * @return
 760              */
 761             //FIXME this method is highly critical, because
 762             //  * it will have serious performance and memory problems with large databases
 763             //        (databases may easily have >1 Mio source records)
 764             //  * it does not make sense to search for existing sources and then clone them
 765             //    we need to search for existing references instead and use them (if exist)
 766             //    for our new source.
 767             protected IdentifiableSource getIdentifiableSource(Reference reference, String citationDetail) {
 768
 769               /*  List<IdentifiableSource> issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
 770
 771
 772                 if (reference != null){
 773                     try {
 774                         for (OriginalSourceBase<?> osb: issTmp){
 775                             if (osb.getCitation() != null && osb.getCitation().getTitleCache().equalsIgnoreCase(reference.getTitleCache())){
 776                                 String osbDetail = osb.getCitationMicroReference();
 777                                 if ((StringUtils.isBlank(osbDetail) && StringUtils.isBlank(citationDetail))
 778                                         || (osbDetail != null && osbDetail.equalsIgnoreCase(citationDetail)) ) {
 779 //                                  System.out.println("REFERENCE FOUND RETURN EXISTING SOURCE");
 780                                     return (IdentifiableSource) osb.clone();
 781                                 }
 782                             }
 783                         }
 784                     } catch (CloneNotSupportedException e) {
 785                         throw new RuntimeException(e);
 786                     } catch (Exception e1){
 787                         e1.printStackTrace();
 788                     }
 789                 }
 790         */
 791                 IdentifiableSource sour = IdentifiableSource.NewInstance(OriginalSourceType.Import,null,null, reference,citationDetail);
 792                 return sour;
 793             }
 794
 795             /**
 796              * Add the hierarchy for a Taxon(add higher taxa)
 797              * @param classification
 798              * @param taxon: a taxon to add as a node
 799              * @param state: the ABCD import state
 800              */
 801             protected void addParentTaxon(Taxon taxon, STATE state, boolean preferredFlag, Classification classification){
 802                 INonViralName  nvname = taxon.getName();
 803                 Rank rank = nvname.getRank();
 804                 Taxon genus =null;
 805                 Taxon subgenus =null;
 806                 Taxon species = null;
 807                 Taxon subspecies = null;
 808                 Taxon parent = null;
 809                 if(rank!=null){
 810                     if (rank.isLower(Rank.GENUS() )){
 811                         String genusOrUninomial = nvname.getGenusOrUninomial();
 812                         TaxonName taxonName = getOrCreateTaxonName(genusOrUninomial, Rank.GENUS(), preferredFlag, state, -1);
 813                         genus = getOrCreateTaxonForName(taxonName, state);
 814                         if (genus == null){
 815                             logger.debug("The genus should not be null " + taxonName);
 816                         }
 817                         if (preferredFlag) {
 818                             parent = linkParentChildNode(null, genus, classification, state);
 819                         }
 820
 821                     }
 822                     if (rank.isLower(Rank.SUBGENUS())){
 823                         String prefix = nvname.getGenusOrUninomial();
 824                         String name = nvname.getInfraGenericEpithet();
 825                         if (name != null){
 826                             TaxonName taxonName = getOrCreateTaxonName(prefix+" "+name, Rank.SUBGENUS(), preferredFlag, state, -1);
 827                             subgenus = getOrCreateTaxonForName(taxonName, state);
 828                             if (preferredFlag) {
 829                                 parent = linkParentChildNode(genus, subgenus, classification, state);
 830                             }            }
 831                     }
 832                     if (rank.isLower(Rank.SPECIES())){
 833                         if (subgenus!=null){
 834                             String prefix = nvname.getGenusOrUninomial();
 835                             String name = nvname.getInfraGenericEpithet();
 836                             String spe = nvname.getSpecificEpithet();
 837                             if (spe != null){
 838                                 TaxonName taxonName = getOrCreateTaxonName(prefix+" "+name+" "+spe, Rank.SPECIES(), preferredFlag, state, -1);
 839                                 species = getOrCreateTaxonForName(taxonName, state);
 840                                 if (preferredFlag) {
 841                                     parent = linkParentChildNode(subgenus, species, classification, state);
 842                                 }
 843                             }
 844                         }
 845                         else{
 846                             String prefix = nvname.getGenusOrUninomial();
 847                             String name = nvname.getSpecificEpithet();
 848                             if (name != null){
 849                                 TaxonName taxonName = getOrCreateTaxonName(prefix+" "+name, Rank.SPECIES(), preferredFlag, state, -1);
 850                                 species = getOrCreateTaxonForName(taxonName, state);
 851                                 if (preferredFlag) {
 852                                     parent = linkParentChildNode(genus, species, classification, state);
 853                                 }
 854                             }
 855                         }
 856                     }
 857                     if (rank.isLower(Rank.INFRASPECIES())){
 858                         TaxonName taxonName = getOrCreateTaxonName(nvname.getFullTitleCache(), Rank.SUBSPECIES(), preferredFlag, state, -1);
 859                         subspecies = getOrCreateTaxonForName(taxonName, state);
 860                         if (preferredFlag) {
 861                             parent = linkParentChildNode(species, subspecies, classification, state);
 862                         }
 863                     }
 864                 }
 865                 if (preferredFlag && parent!=taxon ) {
 866                     linkParentChildNode(parent, taxon, classification, state);
 867                 }
 868             }
 869
 870             /**
 871              * Link a parent to a child and save it in the current classification
 872              * @param parent: the higher Taxon
 873              * @param child : the lower (or current) Taxon
 874              * return the Taxon from the new created Node
 875              * @param classification
 876              * @param state
 877              */
 878             protected Taxon linkParentChildNode(Taxon parent, Taxon child, Classification classification, STATE state) {
 879                 TaxonNode node =null;
 880                 if (parent != null) {
 881                     parent = (Taxon) getTaxonService().find(parent.getUuid());
 882                     child = (Taxon) getTaxonService().find(child.getUuid());
 883                     //here we do not have to check if the taxon nodes already exists
 884                     //this is done by classification.addParentChild()
 885                     //do not add child node if it already exists
 886                     if(hasTaxonNodeInClassification(child, classification)){
 887                         return child;
 888                     }
 889                     else{
 890                         node = classification.addParentChild(parent, child, state.getRef(), "");
 891                         save(node, state);
 892                     }
 893                 }
 894                 else {
 895                     if (child == null){
 896                         logger.debug("The child should not be null!");
 897                     }
 898                     child = (Taxon) getTaxonService().find(child.getUuid());
 899                     //do not add child node if it already exists
 900                     if(hasTaxonNodeInClassification(child, classification)){
 901                         return child;
 902                     }
 903                     else{
 904                         node = classification.addChildTaxon(child, state.getRef(), null);
 905                         save(node, state);
 906                     }
 907                 }
 908                 if(node!=null){
 909                     state.getReport().addTaxonNode(node);
 910                     return node.getTaxon();
 911                 }
 912                 String message = "Could not create taxon node for " +child;
 913                 state.getReport().addInfoMessage(message);
 914                 logger.warn(message);
 915                 return null;
 916             }
 917
 918             protected Taxon getOrCreateTaxonForName(TaxonName taxonName, STATE state){
 919                 if (taxonName != null){
 920                 Set<Taxon> acceptedTaxa = taxonName.getTaxa();
 921                 if(acceptedTaxa.size()>0){
 922                     Taxon firstAcceptedTaxon = acceptedTaxa.iterator().next();
 923                     if(acceptedTaxa.size()>1){
 924                         String message = "More than one accepted taxon was found for taxon name: "
 925                                 + taxonName.getTitleCache() + "!\n" + firstAcceptedTaxon + "was chosen for "+state.getDerivedUnitBase();
 926                         state.getReport().addInfoMessage(message);
 927                         logger.warn(message);
 928                     }
 929                     else{
 930                         return firstAcceptedTaxon;
 931                     }
 932                 }
 933                 else{
 934                     Set<TaxonBase> taxonAndSynonyms = taxonName.getTaxonBases();
 935                     for (TaxonBase taxonBase : taxonAndSynonyms) {
 936                         if(taxonBase.isInstanceOf(Synonym.class)){
 937                             Synonym synonym = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
 938                             Taxon acceptedTaxonOfSynonym = synonym.getAcceptedTaxon();
 939                             if(acceptedTaxonOfSynonym == null){
 940                                 String message = "No accepted taxon could be found for taxon name: "
 941                                         + taxonName.getTitleCache()
 942                                         + "!";
 943                                 state.getReport().addInfoMessage(message);
 944                                 logger.warn(message);
 945                             }
 946                             else{
 947                                 return acceptedTaxonOfSynonym;
 948                             }
 949                         }
 950                     }
 951                 }
 952                 Taxon taxon = Taxon.NewInstance(taxonName, state.getRef());
 953                 save(taxon, state);
 954                 state.getReport().addTaxon(taxon);
 955                 logger.info("Created new taxon "+ taxon);
 956                 return taxon;
 957                 }
 958                 return null;
 959
 960             }
 961
 962             private boolean hasTaxonNodeInClassification(Taxon taxon, Classification classification){
 963                 if(taxon.getTaxonNodes()!=null){
 964                     for (TaxonNode node : taxon.getTaxonNodes()){
 965                         if(node.getClassification().equals(classification)){
 966                             return true;
 967                         }
 968                     }
 969                 }
 970                 return false;
 971             }
 972
 973             /**
 974              * HandleIdentifications : get the scientific names present in the ABCD
 975              * document and store link them with the observation/specimen data
 976              * @param state: the current ABCD import state
 977              * @param derivedUnitFacade : the current derivedunitfacade
 978              */
 979             protected void handleIdentifications(STATE state, DerivedUnitFacade derivedUnitFacade) {
 980                SpecimenImportConfiguratorBase config = state.getConfig();
 981
 982
 983                 String scientificName = "";
 984                 boolean preferredFlag = false;
 985
 986                 if (state.getDataHolder().getNomenclatureCode() == ""){
 987                     if (config.getNomenclaturalCode() != null){
 988                         if (config.getNomenclaturalCode() != null){
 989                             state.getDataHolder().setNomenclatureCode(config.getNomenclaturalCode().toString());
 990
 991                         }
 992                     }
 993                 }
 994
 995                 for (int i = 0; i < state.getDataHolder().getIdentificationList().size(); i++) {
 996                     Identification identification = state.getDataHolder().getIdentificationList().get(i);
 997                     scientificName = identification.getScientificName().replaceAll(" et ", " & ");
 998
 999                     String preferred = identification.getPreferred();
1000                     preferredFlag = false;
1001                     if (preferred != null || state.getDataHolder().getIdentificationList().size()==1){
1002                         if (state.getDataHolder().getIdentificationList().size()==1){
1003                             preferredFlag = true;
1004                         }else if (preferred != null && (preferred.equals("1") || preferred.toLowerCase().indexOf("true") != -1) ) {
1005                         preferredFlag = true;
1006                     }
1007
1008                     }
1009                     if (identification.getCode() != null){
1010                     if (identification.getCode().indexOf(':') != -1) {
1011                         state.getDataHolder().setNomenclatureCode(identification.getCode().split(COLON)[1]);
1012                     }
1013                     else{
1014                         state.getDataHolder().setNomenclatureCode(identification.getCode());
1015                     }
1016                     }
1017                     TaxonName taxonName = getOrCreateTaxonName(scientificName, null, preferredFlag, state, i);
1018                     Taxon taxon = getOrCreateTaxonForName(taxonName, state);
1019                     addTaxonNode(taxon, state,preferredFlag);
1020                     linkDeterminationEvent(state, taxon, preferredFlag, derivedUnitFacade, identification.getIdentifier(), identification.getDate());
1021                 }
1022             }
1023
1024             /**
1025              * @param taxon : a taxon to add as a node
1026              * @param state : the ABCD import state
1027              */
1028             protected void addTaxonNode(Taxon taxon, STATE state, boolean preferredFlag) {
1029                 SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
1030                 logger.info("link taxon to a taxonNode "+taxon.getTitleCache());
1031                 //only add nodes if not already existing in current classification or default classification
1032
1033                 //check if node exists in current classification
1034                 //NOTE: we cannot use hasTaxonNodeInClassification() here because we are first creating it here
1035                 if (!existsInClassification(taxon,state.getClassification(), state)){
1036                     if(config.isMoveNewTaxaToDefaultClassification()){
1037                         //check if node exists in default classification
1038                         if (!existsInClassification(taxon, state.getDefaultClassification(true), state)){
1039                             addParentTaxon(taxon, state, preferredFlag, state.getDefaultClassification(true));
1040                         }
1041                     }else{
1042                         //add non-existing taxon to current classification
1043                         addParentTaxon(taxon, state, preferredFlag, state.getClassification());
1044                     }
1045
1046                 }
1047             }
1048
1049
1050             private boolean existsInClassification(Taxon taxon, Classification classification, STATE state){
1051                 boolean exist = false;
1052                 ICdmRepository cdmAppController = state.getConfig().getCdmAppController();
1053             if(cdmAppController==null){
1054                 cdmAppController = this;
1055             }
1056             if (classification != null){
1057                 if (!taxon.getTaxonNodes().isEmpty()){
1058                     for (TaxonNode node:taxon.getTaxonNodes()){
1059                         if (node.getClassification().equals(classification)){
1060                             return true;
1061                         }
1062                     }
1063                 }
1064 // we do not need this because we already searched for taxa in db in the previous steps
1065 //              List<UuidAndTitleCache<TaxonNode>> uuidAndTitleCacheOfAllTaxa = cdmAppController.getClassificationService().getTaxonNodeUuidAndTitleCacheOfAcceptedTaxaByClassification(classification.getUuid());
1066 //              if (uuidAndTitleCacheOfAllTaxa != null){
1067 //                      for (UuidAndTitleCache p : uuidAndTitleCacheOfAllTaxa){
1068 //                          try{
1069 //                              if(p.getTitleCache().equals(taxon.getTitleCache())) {
1070 //                                  exist = true;
1071 //                              }
1072 //                          }
1073 //                          catch(Exception e){
1074 //                              logger.warn("TaxonNode doesn't seem to have a taxon");
1075 //                          }
1076 //                      }
1077 //              }
1078             }
1079                 return exist;
1080             }
1081
1082             /**
1083              * join DeterminationEvent to the Taxon Object
1084              * @param state : the ABCD import state
1085              * @param taxon: the current Taxon
1086              * @param preferredFlag :if the current name is preferred
1087              * @param derivedFacade : the derived Unit Facade
1088              */
1089             @SuppressWarnings("rawtypes")
1090         protected void linkDeterminationEvent(STATE state, Taxon taxon, boolean preferredFlag,  DerivedUnitFacade derivedFacade, String identifierStr, String dateStr) {
1091                 SpecimenImportConfiguratorBase config = state.getConfig();
1092                 if (logger.isDebugEnabled()){
1093                     logger.info("start linkdetermination with taxon:" + taxon.getUuid()+", "+taxon);
1094                 }
1095
1096                 DeterminationEvent determinationEvent = DeterminationEvent.NewInstance();
1097                 //determinationEvent.setTaxon(taxon);
1098                 determinationEvent.setTaxonName(taxon.getName());
1099                 determinationEvent.setPreferredFlag(preferredFlag);
1100
1101
1102                 determinationEvent.setIdentifiedUnit(state.getDerivedUnitBase());
1103                 if (state.getPersonStore().get(identifierStr) != null){
1104                     determinationEvent.setActor((AgentBase)state.getPersonStore().get(identifierStr));
1105                 } else if (identifierStr != null){
1106                     Person identifier = Person.NewTitledInstance(identifierStr);
1107                     determinationEvent.setActor(identifier);
1108                 }
1109                 if (dateStr != null){
1110                     determinationEvent.setTimeperiod(TimePeriodParser.parseString(dateStr));
1111                 }
1112                 state.getDerivedUnitBase().addDetermination(determinationEvent);
1113
1114                 if (logger.isDebugEnabled()){
1115                     logger.debug("NB TYPES INFO: "+ state.getDataHolder().getStatusList().size());
1116                 }
1117                 for (SpecimenTypeDesignationStatus specimenTypeDesignationstatus : state.getDataHolder().getStatusList()) {
1118                     if (specimenTypeDesignationstatus != null) {
1119                         if (logger.isDebugEnabled()){
1120                             logger.debug("specimenTypeDesignationstatus :"+ specimenTypeDesignationstatus);
1121                         }
1122
1123                         ICdmRepository cdmAppController = config.getCdmAppController();
1124                         if(cdmAppController == null){
1125                             cdmAppController = this;
1126                         }
1127                         specimenTypeDesignationstatus = HibernateProxyHelper.deproxy(cdmAppController.getTermService().find(specimenTypeDesignationstatus.getUuid()), SpecimenTypeDesignationStatus.class);
1128                         //Designation
1129                         TaxonName name = taxon.getName();
1130                         SpecimenTypeDesignation designation = SpecimenTypeDesignation.NewInstance();
1131
1132                         designation.setTypeStatus(specimenTypeDesignationstatus);
1133                         designation.setTypeSpecimen(state.getDerivedUnitBase());
1134                         name.addTypeDesignation(designation, true);
1135                     }
1136                 }
1137                 save(state.getDerivedUnitBase(), state);
1138
1139                 for (String[] fullReference : state.getDataHolder().getReferenceList()) {
1140
1141
1142                     String strReference=fullReference[0];
1143                     String citationDetail = fullReference[1];
1144                     String citationURL = fullReference[2];
1145                     List<Reference> references = getReferenceService().listByTitleWithRestrictions(Reference.class, "strReference", MatchMode.EXACT, null, null, null, null, null);
1146
1147                     if (!references.isEmpty()){
1148                         Reference reference = null;
1149                         for (Reference refe: references) {
1150                             if (refe.getTitleCache().equalsIgnoreCase(strReference)) {
1151                                 reference =refe;
1152                                 break;
1153                             }
1154                         }
1155                         if (reference ==null){
1156                             reference = ReferenceFactory.newGeneric();
1157                             reference.setTitleCache(strReference, true);
1158                             save(reference, state);
1159                         }
1160                         determinationEvent.addReference(reference);
1161                     }
1162                 }
1163                 save(state.getDerivedUnitBase(), state);
1164
1165                 if (config.isAddIndividualsAssociationsSuchAsSpecimenAndObservations() && preferredFlag) {
1166                     //do not add IndividualsAssociation to non-preferred taxa
1167                     if (logger.isDebugEnabled()){
1168                         logger.debug("isDoCreateIndividualsAssociations");
1169                     }
1170
1171                     makeIndividualsAssociation(state, taxon, determinationEvent);
1172
1173                     save(state.getDerivedUnitBase(), state);
1174                 }
1175             }
1176
1177             /**
1178              * create and link each association (specimen, observation..) to the accepted taxon
1179              * @param state : the ABCD import state
1180              * @param taxon: the current Taxon
1181              * @param determinationEvent:the determinationevent
1182              */
1183             protected void makeIndividualsAssociation(STATE state, Taxon taxon, DeterminationEvent determinationEvent) {
1184                 SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
1185                 SpecimenUserInteraction sui = config.getSpecimenUserInteraction();
1186
1187                 if (logger.isDebugEnabled()){
1188                     logger.info("MAKE INDIVIDUALS ASSOCIATION");
1189                 }
1190
1191                 TaxonDescription taxonDescription = null;
1192                 Set<TaxonDescription> descriptions= taxon.getDescriptions();
1193                if (!descriptions.isEmpty()){ taxonDescription = descriptions.iterator().next();}
1194
1195 //                  for (TaxonDescription description : descriptions){
1196 //                      Set<IdentifiableSource> sources =  new HashSet<>();
1197 //                      sources.addAll(description.getTaxon().getSources());
1198 //                      sources.addAll(description.getSources());
1199 //                      for (IdentifiableSource source:sources){
1200 //                          if(state.getRef().equals(source.getCitation())) {
1201 //                              taxonDescription = description;
1202 //                          }
1203 //                      }
1204 //                  }
1205               //  }
1206                 if (taxonDescription == null){
1207                     taxonDescription = TaxonDescription.NewInstance(taxon, false);
1208                     if(sourceNotLinkedToElement(taxonDescription,state.getRef(),null)) {
1209                         taxonDescription.addSource(OriginalSourceType.Import, null, null, state.getRef(), null);
1210                     }
1211                     state.setDescriptionGroup(taxonDescription);
1212                     taxon.addDescription(taxonDescription);
1213                 }
1214
1215                 //PREPARE REFERENCE QUESTIONS
1216
1217                 Map<String,OriginalSourceBase<?>> sourceMap = new HashMap<String, OriginalSourceBase<?>>();
1218
1219                 List<IdentifiableSource> issTmp = new ArrayList<>();//getCommonService().list(IdentifiableSource.class, null, null, null, null);
1220                 List<DescriptionElementSource> issTmp2 = new ArrayList<>();//getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1221
1222                 Set<OriginalSourceBase> osbSet = new HashSet<OriginalSourceBase>();
1223                 if(issTmp2!=null) {
1224                     osbSet.addAll(issTmp2);
1225                 }
1226                 if(issTmp!=null) {
1227                     osbSet.addAll(issTmp);
1228                 }
1229
1230
1231                 addToSourceMap(sourceMap, osbSet);
1232
1233 //              if (((Abcd206ImportConfigurator) state.getConfig()).isInteractWithUser()){
1234 //                  List<OriginalSourceBase<?>> res = null;
1235 //                  if(!state.isDescriptionSourcesSet()){
1236 //                      res = sui.askForSource(sourceMap, "the description group ("+taxon+")",
1237 //                              "The current reference is "+state.getRef().getTitleCache(),getReferenceService(), state.getDataHolder().docSources);
1238 //                      state.setDescriptionRefs(res);
1239 //                      state.setDescriptionSourcesSet(true);
1240 //                  }
1241 //                  else{
1242 //                      res=state.getDescriptionRefs();
1243 //                  }
1244 //                  if(res !=null) {
1245 //                      for (OriginalSourceBase<?> sour:res){
1246 //                          if(sour.isInstanceOf(IdentifiableSource.class)){
1247 //                              try {
1248 //                                  if(sourceNotLinkedToElement(taxonDescription,sour)) {
1249 //                                      taxonDescription.addSource((IdentifiableSource)sour.clone());
1250 //                                  }
1251 //                              } catch (CloneNotSupportedException e) {
1252 //                                  logger.warn("no cloning?");
1253 //                              }
1254 //                          }else{
1255 //                              if(sourceNotLinkedToElement(taxonDescription,sour)) {
1256 //                                  taxonDescription.addSource(OriginalSourceType.Import,null, null, sour.getCitation(),sour.getCitationMicroReference());
1257 //                              }
1258 //                          }
1259 //                      }
1260 //                  }
1261 //              }
1262 //              else {
1263                     if(sourceNotLinkedToElement(taxonDescription,state.getRef(),null)) {
1264                         taxonDescription.addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1265                     }
1266 //              }
1267                 state.setDescriptionGroup(taxonDescription);
1268
1269                 IndividualsAssociation indAssociation = IndividualsAssociation.NewInstance();
1270                 Feature feature = makeFeature(state.getDerivedUnitBase());
1271                 indAssociation.setAssociatedSpecimenOrObservation(state.getDerivedUnitBase());
1272                 indAssociation.setFeature(feature);
1273 //
1274 //              if (((Abcd206ImportConfigurator) state.getConfig()).isInteractWithUser()){
1275 //                  sourceMap = new HashMap<String, OriginalSourceBase<?>>();
1276 //
1277 //                  issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
1278 //                  issTmp2 = getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1279 //
1280 //                  osbSet = new HashSet<OriginalSourceBase>();
1281 //                  if(issTmp2!=null) {
1282 //                      osbSet.addAll(issTmp2);
1283 //                  }
1284 //                  if(issTmp!=null) {
1285 //                      osbSet.addAll(issTmp);
1286 //                  }
1287 //
1288 //
1289 //                  addToSourceMap(sourceMap, osbSet);
1290 //
1291 //                  List<OriginalSourceBase<?>> sources =null;
1292 //                  if(!state.isAssociationSourcesSet()) {
1293 //                      sources = sui.askForSource(sourceMap,  "descriptive element (association) ",taxon.toString(),
1294 //                              getReferenceService(),state.getDataHolder().getDocSources());
1295 //                      state.setAssociationRefs(sources);
1296 //                      state.setAssociationSourcesSet(true);
1297 //                  }
1298 //                  else{
1299 //                      sources=state.getAssociationRefs();
1300 //                  }
1301 //                  if(sources !=null) {
1302 //                      for (OriginalSourceBase<?> source: sources) {
1303 //                          if(source !=null) {
1304 //                              if(source.isInstanceOf(DescriptionElementSource.class)){
1305 //                                  try {
1306 //                                      if(sourceNotLinkedToElement(indAssociation,source)) {
1307 //                                          indAssociation.addSource((DescriptionElementSource)source.clone());
1308 //                                      }
1309 //                                  } catch (CloneNotSupportedException e) {
1310 //                                      logger.warn("clone forbidden?");
1311 //                                  }
1312 //                              }else{
1313 //                                  if(sourceNotLinkedToElement(indAssociation,source)) {
1314 //                                      indAssociation.addSource(OriginalSourceType.Import,null, null, source.getCitation(),source.getCitationMicroReference());
1315 //                                  }
1316 //                                  try {
1317 //                                      if(sourceNotLinkedToElement(state.getDerivedUnitBase(), source)) {
1318 //                                          state.getDerivedUnitBase().addSource((IdentifiableSource) source.clone());
1319 //                                      }
1320 //                                  } catch (CloneNotSupportedException e) {
1321 //                                      // TODO Auto-generated catch block
1322 //                                      e.printStackTrace();
1323 //                                  }
1324 //                              }
1325 //
1326 //                          }
1327 //                      }
1328 //                  }
1329 //              }else {
1330                     if(sourceNotLinkedToElement(indAssociation,state.getRef(),null)) {
1331                         indAssociation.addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1332                     }
1333                     if(sourceNotLinkedToElement(state.getDerivedUnitBase(), state.getRef(),null)) {
1334                         state.getDerivedUnitBase().addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1335                     }
1336                     for (Reference citation : determinationEvent.getReferences()) {
1337                         if(sourceNotLinkedToElement(indAssociation,citation,null))
1338                         {
1339                             indAssociation.addSource(DescriptionElementSource.NewInstance(OriginalSourceType.Import, null, null, citation, null));
1340                         }
1341                         if(sourceNotLinkedToElement(state.getDerivedUnitBase(), state.getRef(),null)) {
1342                             state.getDerivedUnitBase().addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1343                         }
1344                     }
1345            //     }
1346
1347                 taxonDescription.addElement(indAssociation);
1348
1349                 save(taxonDescription, state);
1350                 save(taxon, state);
1351                 state.getReport().addDerivate(state.getDerivedUnitBase(), config);
1352                 state.getReport().addIndividualAssociation(taxon, state.getDataHolder().getUnitID(), state.getDerivedUnitBase());
1353             }
1354
1355             /**
1356              * @param derivedUnitBase2
1357              * @param ref2
1358              * @param object
1359              * @return
1360              */
1361             private boolean sourceNotLinkedToElement(DerivedUnit derivedUnitBase2, Reference b, String d) {
1362                 Set<IdentifiableSource> linkedSources = derivedUnitBase2.getSources();
1363                 for (IdentifiableSource is:linkedSources){
1364                     Reference a = is.getCitation();
1365                     String c = is.getCitationMicroReference();
1366
1367                     boolean refMatch=false;
1368                     boolean microMatch=false;
1369
1370                     try{
1371                         if (a==null && b==null) {
1372                             refMatch=true;
1373                         }
1374                         if (a!=null && b!=null) {
1375                             if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1376                                 refMatch=true;
1377                             }
1378                         }
1379                     }catch(Exception e){}
1380
1381
1382                     try{
1383                         if (c==null && d==null) {
1384                             microMatch=true;
1385                         }
1386                         if(c!=null && d!=null) {
1387                             if(c.equalsIgnoreCase(d)) {
1388                                 microMatch=true;
1389                             }
1390                         }
1391                     }
1392                     catch(Exception e){}
1393
1394                     if (microMatch && refMatch) {
1395                         return false;
1396                     }
1397
1398
1399                 }
1400                 return true;
1401             }
1402
1403             private <T extends OriginalSourceBase<?>> boolean  sourceNotLinkedToElement(ISourceable<T> sourcable, Reference reference, String microReference) {
1404                 Set<T> linkedSources = sourcable.getSources();
1405                 for (T is:linkedSources){
1406                     Reference unitReference = is.getCitation();
1407                     String unitMicroReference = is.getCitationMicroReference();
1408
1409                     boolean refMatch=false;
1410                     boolean microMatch=false;
1411
1412                     try{
1413                         if (unitReference==null && reference==null) {
1414                             refMatch=true;
1415                         }
1416                         if (unitReference!=null && reference!=null) {
1417                             if (unitReference.getTitleCache().equalsIgnoreCase(reference.getTitleCache())) {
1418                                 refMatch=true;
1419                             }
1420                         }
1421                     }catch(Exception e){}
1422
1423                     try{
1424                         if (unitMicroReference==null && microReference==null) {
1425                             microMatch=true;
1426                         }
1427                         if(unitMicroReference!=null && microReference!=null) {
1428                             if(unitMicroReference.equalsIgnoreCase(microReference)) {
1429                                 microMatch=true;
1430                             }
1431                         }
1432                     }
1433                     catch(Exception e){}
1434
1435                     if (microMatch && refMatch) {
1436                         return false;
1437                     }
1438                 }
1439                 return true;
1440             }
1441
1442             /**
1443              * look for the Feature object (FieldObs, Specimen,...)
1444              * @param unit : a specimen or obersvation base
1445              * @return the corresponding Feature
1446              */
1447             private Feature makeFeature(SpecimenOrObservationBase<?> unit) {
1448                 SpecimenOrObservationType type = unit.getRecordBasis();
1449
1450
1451
1452                 if (type.isFeatureObservation()){
1453                     return Feature.OBSERVATION();
1454                 }else if (type.isFeatureSpecimen()){
1455                     return Feature.SPECIMEN();
1456                 }else if (type == SpecimenOrObservationType.DerivedUnit){
1457                     return Feature.OBSERVATION();
1458                     //            return getFeature("Specimen or observation");
1459                 }else{
1460                     String message = "Unhandled record basis '%s' for defining individuals association feature type. Use default.";
1461                     logger.warn(String.format(message, type.getMessage()));
1462                     return Feature.OBSERVATION();
1463                     //            return getFeature("Specimen or observation");
1464
1465                 }
1466             }
1467
1468
1469             /**
1470              * @param sourceMap
1471              * @param osbSet
1472              */
1473             protected void addToSourceMap(Map<String, OriginalSourceBase<?>> sourceMap, Set<OriginalSourceBase> osbSet) {
1474                 for( OriginalSourceBase<?> osb:osbSet) {
1475                     if(osb.getCitation()!=null && osb.getCitationMicroReference() !=null  && !osb.getCitationMicroReference().isEmpty()) {
1476                         try{
1477                             sourceMap.put(osb.getCitation().getTitleCache()+ "---"+osb.getCitationMicroReference(),osb);
1478                         }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1479                     } else if(osb.getCitation()!=null){
1480                         try{
1481                             sourceMap.put(osb.getCitation().getTitleCache(),osb);
1482                         }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1483                     }
1484                 }
1485             }
1486
1487
1488 }