cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/SpecimenImportBase.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.specimen;
  11
  12
  13 import java.util.ArrayList;
  14 import java.util.HashMap;
  15 import java.util.HashSet;
  16 import java.util.List;
  17 import java.util.Map;
  18 import java.util.Set;
  19 import java.util.UUID;
  20
  21 import org.apache.log4j.Logger;
  22
  23 import eu.etaxonomy.cdm.api.application.ICdmRepository;
  24 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
  25 import eu.etaxonomy.cdm.api.service.config.FindOccurrencesConfigurator;
  26 import eu.etaxonomy.cdm.api.service.pager.Pager;
  27 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  28 import eu.etaxonomy.cdm.io.common.CdmImportBase;
  29 import eu.etaxonomy.cdm.io.common.IImportConfigurator;
  30 import eu.etaxonomy.cdm.io.specimen.abcd206.in.Identification;
  31 import eu.etaxonomy.cdm.io.specimen.abcd206.in.SpecimenImportReport;
  32 import eu.etaxonomy.cdm.model.agent.AgentBase;
  33 import eu.etaxonomy.cdm.model.agent.Institution;
  34 import eu.etaxonomy.cdm.model.agent.Person;
  35 import eu.etaxonomy.cdm.model.agent.Team;
  36 import eu.etaxonomy.cdm.model.common.CdmBase;
  37 import eu.etaxonomy.cdm.model.common.ISourceable;
  38 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  39 import eu.etaxonomy.cdm.model.common.LanguageString;
  40 import eu.etaxonomy.cdm.model.common.OriginalSourceBase;
  41 import eu.etaxonomy.cdm.model.common.OriginalSourceType;
  42 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  43 import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
  44 import eu.etaxonomy.cdm.model.description.Feature;
  45 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  46 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  47 import eu.etaxonomy.cdm.model.name.BacterialName;
  48 import eu.etaxonomy.cdm.model.name.BotanicalName;
  49 import eu.etaxonomy.cdm.model.name.CultivarPlantName;
  50 import eu.etaxonomy.cdm.model.name.INonViralName;
  51 import eu.etaxonomy.cdm.model.name.ITaxonNameBase;
  52 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  53 import eu.etaxonomy.cdm.model.name.NonViralName;
  54 import eu.etaxonomy.cdm.model.name.Rank;
  55 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
  56 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
  57 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
  58 import eu.etaxonomy.cdm.model.name.TaxonNameFactory;
  59 import eu.etaxonomy.cdm.model.occurrence.Collection;
  60 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  61 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
  62 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
  63 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationType;
  64 import eu.etaxonomy.cdm.model.reference.Reference;
  65 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  66 import eu.etaxonomy.cdm.model.taxon.Classification;
  67 import eu.etaxonomy.cdm.model.taxon.Synonym;
  68 import eu.etaxonomy.cdm.model.taxon.Taxon;
  69 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  70 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  71 import eu.etaxonomy.cdm.persistence.query.MatchMode;
  72 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
  73 import eu.etaxonomy.cdm.strategy.parser.ParserProblem;
  74 import eu.etaxonomy.cdm.strategy.parser.TimePeriodParser;
  75
  76
  77 /**
  78  * @author p.kelbert
  79  * @created 20.10.2008
  80  */
  81 public abstract class SpecimenImportBase<CONFIG extends IImportConfigurator, STATE extends SpecimenImportStateBase>  extends CdmImportBase<CONFIG, STATE> {
  82
  83     private static final long serialVersionUID = 4423065367998125678L;
  84     private static final Logger logger = Logger.getLogger(SpecimenImportBase.class);
  85         protected final boolean DEBUG = true;
  86
  87         protected static final UUID SPECIMEN_SCAN_TERM = UUID.fromString("acda15be-c0e2-4ea8-8783-b9b0c4ad7f03");
  88
  89         private static final String COLON = ":";
  90
  91
  92         @Override
  93     protected abstract void doInvoke(STATE state);
  94
  95         /**
  96          * Handle a single unit
  97          * @param state
  98          * @param item
  99          */
 100         protected abstract void handleSingleUnit(STATE state, Object item) ;
 101
 102
 103
 104         protected TaxonNameBase<?, ?> getOrCreateTaxonName(String scientificName, Rank rank, boolean preferredFlag, STATE state, int unitIndexInAbcdFile){
 105             TaxonNameBase<?, ?> taxonName = null;
 106         SpecimenImportConfiguratorBase<?,?> config = state.getConfig();
 107
 108         //check atomised name data for rank
 109         //new name will be created
 110         ITaxonNameBase atomisedTaxonName = null;
 111         if (rank==null && unitIndexInAbcdFile>=0 && ((state.getDataHolder().getAtomisedIdentificationList() != null && !state.getDataHolder().getAtomisedIdentificationList().isEmpty())|| state.getDataHolder().getAtomisedIdentificationList().size() > 0)) {
 112             atomisedTaxonName = setTaxonNameByType(state.getDataHolder().getAtomisedIdentificationList().get(unitIndexInAbcdFile), scientificName, state);
 113             if(atomisedTaxonName!=null){
 114                 rank = atomisedTaxonName.getRank();
 115             }
 116         }
 117         if(config.isReuseExistingTaxaWhenPossible()){
 118             ITaxonNameBase parsedName = atomisedTaxonName;
 119             if(parsedName==null){
 120
 121                 parsedName = parseScientificName(scientificName, state, state.getReport(), rank);
 122
 123             }
 124             atomisedTaxonName = parsedName;
 125             if(config.isIgnoreAuthorship() && parsedName!=null){// && preferredFlag){
 126                 // do not ignore authorship for non-preferred names because they need
 127                 // to be created for the determination history
 128                 String nameCache = TaxonNameBase.castAndDeproxy(parsedName).getNameCache();
 129                 List<NonViralName> names = getNameService().findNamesByNameCache(nameCache, MatchMode.EXACT, null);
 130                 if (!names.isEmpty()){
 131                      taxonName = getBestMatchingName(scientificName, new ArrayList<TaxonNameBase>(names), state);
 132                 }
 133                 if (taxonName == null && !names.isEmpty()){
 134                     taxonName = names.get(0);
 135                 }
 136
 137             } else {
 138                 //search for existing names
 139                 List<TaxonNameBase> names = getNameService().listByTitle(TaxonNameBase.class, scientificName, MatchMode.EXACT, null, null, null, null, null);
 140                 taxonName = getBestMatchingName(scientificName, names, state);
 141                 //still nothing found -> try with the atomised name full title cache
 142                 if(taxonName==null && atomisedTaxonName!=null){
 143                     names = getNameService().listByTitle(TaxonNameBase.class, atomisedTaxonName.getFullTitleCache(), MatchMode.EXACT, null, null, null, null, null);
 144                     taxonName = getBestMatchingName(atomisedTaxonName.getTitleCache(), names, state);
 145                     //still nothing found -> try with the atomised name title cache
 146                     if(taxonName==null){
 147                         names = getNameService().listByTitle(TaxonNameBase.class, atomisedTaxonName.getTitleCache(), MatchMode.EXACT, null, null, null, null, null);
 148                         taxonName = getBestMatchingName(atomisedTaxonName.getTitleCache(), names, state);
 149                     }
 150                 }
 151
 152             }
 153
 154         }
 155
 156         if(taxonName==null && atomisedTaxonName!=null){
 157             taxonName = (TaxonNameBase<?, ?>) atomisedTaxonName;
 158             state.getReport().addName(taxonName);
 159             logger.info("Created new taxon name "+taxonName);
 160             if(taxonName.hasProblem()){
 161                 state.getReport().addInfoMessage(String.format("Created %s with parsing problems", taxonName));
 162             }
 163             if(!atomisedTaxonName.getTitleCache().equals(scientificName)){
 164                 state.getReport().addInfoMessage(String.format("Taxon %s was parsed as %s", scientificName, atomisedTaxonName.getTitleCache()));
 165             }
 166         }
 167         else if(taxonName==null){
 168             //create new taxon name
 169
 170             if (state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNAFP)){
 171                 taxonName = TaxonNameFactory.NewBotanicalInstance(rank);
 172             }else if (state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICZN)){
 173                 taxonName = TaxonNameFactory.NewZoologicalInstance(rank);
 174             }else{
 175                 taxonName = TaxonNameFactory.NewNonViralInstance(rank);
 176             }
 177             taxonName.setFullTitleCache(scientificName,true);
 178             taxonName.setTitleCache(scientificName, true);
 179             state.getReport().addName(taxonName);
 180             logger.info("Created new taxon name "+taxonName);
 181         }
 182         save(taxonName, state);
 183         return taxonName;
 184     }
 185
 186          protected TaxonNameBase<?, ?> getBestMatchingName(String scientificName, java.util.Collection<TaxonNameBase> names, STATE state){
 187                 Set<TaxonNameBase> namesWithAcceptedTaxa = new HashSet<TaxonNameBase>();
 188                 List<TaxonNameBase> namesWithAcceptedTaxaInClassification = new ArrayList<TaxonNameBase>();
 189                 for (TaxonNameBase name : names) {
 190                     if(!name.getTaxa().isEmpty()){
 191                         Set<Taxon> taxa = name.getTaxa();
 192                         for (Taxon taxon:taxa){
 193                             if (!taxon.getTaxonNodes().isEmpty()){
 194                                 //use only taxa included in a classification
 195                                 for (TaxonNode node:taxon.getTaxonNodes()){
 196                                     if (state.getClassification() != null && node.getClassification().equals(state.getClassification())){
 197                                         namesWithAcceptedTaxaInClassification.add(name);
 198                                     }else {
 199                                         namesWithAcceptedTaxa.add(name);
 200                                     }
 201                                 }
 202
 203                             }
 204                         }
 205
 206                     }
 207                 }
 208                 String message = String.format("More than one taxon name was found for %s, maybe in other classifications!", scientificName);
 209                 //check for names with accepted taxa in classification
 210                 if(namesWithAcceptedTaxaInClassification.size()>0){
 211                 if(namesWithAcceptedTaxaInClassification.size()>1){
 212
 213                     state.getReport().addInfoMessage(message);
 214                     logger.warn(message);
 215                     return null;
 216                 }
 217                 return namesWithAcceptedTaxaInClassification.iterator().next();
 218             }
 219               //check for any names with accepted taxa
 220                 if(namesWithAcceptedTaxa.size()>0){
 221                     if(namesWithAcceptedTaxa.size()>1){
 222
 223                         state.getReport().addInfoMessage(message);
 224                         logger.warn(message);
 225                         return null;
 226                     }
 227                     return namesWithAcceptedTaxa.iterator().next();
 228                 }
 229                 //no names with accepted taxa found -> check accepted taxa of synonyms
 230                 List<Taxon> taxaFromSynonyms = new ArrayList<>();
 231                 for (TaxonNameBase name : names) {
 232                     Set<TaxonBase> taxonBases = name.getTaxonBases();
 233                     for (TaxonBase taxonBase : taxonBases) {
 234                         if(taxonBase.isInstanceOf(Synonym.class)){
 235                             Synonym synonym = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
 236                             taxaFromSynonyms.add(synonym.getAcceptedTaxon());
 237                         }
 238                     }
 239                 }
 240                 if(taxaFromSynonyms.size()>0){
 241                     if(taxaFromSynonyms.size()>1){
 242                         state.getReport().addInfoMessage(message);
 243                         logger.warn(message);
 244                         return null;
 245                     }
 246                     return taxaFromSynonyms.iterator().next().getName();
 247                 }
 248                 //no accepted and no synonyms -> return one of the names and create a new taxon
 249                 if (names.isEmpty()){
 250                     return null;
 251                 }else{
 252                     return names.iterator().next();
 253                 }
 254             }
 255          /**
 256              * Parse automatically the scientific name
 257              * @param scientificName the scientific name to parse
 258              * @param state the current import state
 259              * @param report the import report
 260              * @return a parsed name
 261              */
 262
 263             protected ITaxonNameBase parseScientificName(String scientificName, STATE state, SpecimenImportReport report, Rank rank) {
 264
 265                 NonViralNameParserImpl nvnpi = NonViralNameParserImpl.NewInstance();
 266                 ITaxonNameBase taxonName = null;
 267                 boolean problem = false;
 268
 269                 if(DEBUG){
 270                     logger.info("parseScientificName " + state.getDataHolder().getNomenclatureCode().toString());
 271                 }
 272
 273                 if (state.getDataHolder().getNomenclatureCode().toString().equals("Zoological") || state.getDataHolder().getNomenclatureCode().toString().contains("ICZN")) {
 274                     taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICZN, rank);
 275                     if (taxonName.hasProblem()) {
 276                         problem = true;
 277                     }
 278                 }
 279                 else if (state.getDataHolder().getNomenclatureCode().toString().equals("Botanical") || state.getDataHolder().getNomenclatureCode().toString().contains("ICBN")) {
 280                     taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNAFP, rank);
 281                     if (taxonName.hasProblem()) {
 282                         problem = true;
 283                     }
 284                 }
 285                 else if (state.getDataHolder().getNomenclatureCode().toString().equals("Bacterial") || state.getDataHolder().getNomenclatureCode().toString().contains("ICBN")) {
 286                     taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNB, rank);
 287                     if (taxonName.hasProblem()) {
 288                         problem = true;
 289                     }
 290                 }
 291                 else if (state.getDataHolder().getNomenclatureCode().toString().equals("Cultivar") || state.getDataHolder().getNomenclatureCode().toString().contains("ICNCP")) {
 292                     taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNCP, rank);
 293                     if (taxonName.hasProblem()) {
 294                         problem = true;
 295                     }
 296                 }
 297                 if (problem) {
 298                     String message = String.format("Parsing problems for %s", scientificName);
 299                     if(taxonName!=null){
 300                         for (ParserProblem parserProblem : taxonName.getParsingProblems()) {
 301                             message += "\n\t- "+parserProblem;
 302                         }
 303                     }
 304                     report.addInfoMessage(message);
 305                     logger.info(message);
 306                 }
 307                 return taxonName;
 308
 309             }
 310
 311             /**
 312              * Create the name without automatic parsing, either because it failed, or because the user deactivated it.
 313              * The name is built upon the ABCD fields
 314              * @param atomisedMap : the ABCD atomised fields
 315              * @param fullName : the full scientific name
 316              * @param state
 317              * @return the corresponding Botanical or Zoological or... name
 318              */
 319             protected TaxonNameBase<?,?> setTaxonNameByType(
 320                     HashMap<String, String> atomisedMap, String fullName, STATE state) {
 321                 boolean problem = false;
 322                 if(DEBUG) {
 323                     logger.info("settaxonnamebytype " + state.getDataHolder().getNomenclatureCode().toString());
 324                 }
 325
 326                 if (state.getDataHolder().getNomenclatureCode().equals("Zoological") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICZN.getUuid())) {
 327                     TaxonNameBase<?,?> taxonName = TaxonNameFactory.NewZoologicalInstance(null);
 328                     taxonName.setFullTitleCache(fullName, true);
 329                     taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
 330                     taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
 331                     taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap,"SpeciesEpithet")));
 332                     taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap,"SubspeciesEpithet")));
 333
 334                     if (taxonName.getGenusOrUninomial() != null){
 335                         taxonName.setRank(Rank.GENUS());
 336                     }
 337
 338                     if (taxonName.getInfraGenericEpithet() != null){
 339                         taxonName.setRank(Rank.SUBGENUS());
 340                     }
 341
 342                     if (taxonName.getSpecificEpithet() != null){
 343                         taxonName.setRank(Rank.SPECIES());
 344                     }
 345
 346                     if (taxonName.getInfraSpecificEpithet() != null){
 347                         taxonName.setRank(Rank.SUBSPECIES());
 348                     }
 349
 350                     Team team = null;
 351                     if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 352                         team = Team.NewInstance();
 353                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
 354                     }
 355                     else {
 356                         if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 357                             team = Team.NewInstance();
 358                             team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
 359                         }
 360                     }
 361                     if (team != null) {
 362                         taxonName.setBasionymAuthorship(team);
 363                     }
 364                     else {
 365                         if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 366                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
 367                         }
 368                         else if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 369                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamAndYear"));
 370                         }
 371                     }
 372                     if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
 373                         team = Team.NewInstance();
 374                         team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
 375                         taxonName.setCombinationAuthorship(team);
 376                     }
 377                     if (taxonName.hasProblem()) {
 378                         logger.info("pb ICZN");
 379                         problem = true;
 380                     }
 381                     else {
 382                         return taxonName;
 383                     }
 384                 }
 385                 else if (state.getDataHolder().getNomenclatureCode().equals("Botanical") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNAFP.getUuid())) {
 386                     BotanicalName taxonName = (BotanicalName) parseScientificName(fullName, state, state.getReport(), null);
 387                     if (taxonName != null){
 388                         return taxonName;
 389                     }
 390                     else{
 391                         taxonName = TaxonNameFactory.NewBotanicalInstance(null);
 392                     }
 393                     taxonName.setFullTitleCache(fullName, true);
 394                     taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
 395                     taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "FirstEpithet")));
 396                     taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "InfraSpeEpithet")));
 397                     try {
 398                         taxonName.setRank(Rank.getRankByName(getFromMap(atomisedMap, "Rank")));
 399                     } catch (Exception e) {
 400                         if (taxonName.getInfraSpecificEpithet() != null){
 401                             taxonName.setRank(Rank.SUBSPECIES());
 402                         }
 403                         else if (taxonName.getSpecificEpithet() != null){
 404                             taxonName.setRank(Rank.SPECIES());
 405                         }
 406                         else if (taxonName.getInfraGenericEpithet() != null){
 407                             taxonName.setRank(Rank.SUBGENUS());
 408                         }
 409                         else if (taxonName.getGenusOrUninomial() != null){
 410                             taxonName.setRank(Rank.GENUS());
 411                         }
 412                     }
 413                     Team team = null;
 414                     if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 415                         team = Team.NewInstance();
 416                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
 417                         taxonName.setBasionymAuthorship(team);
 418                     }
 419                     if (getFromMap(atomisedMap, "AuthorTeam") != null) {
 420                         team = Team.NewInstance();
 421                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeam"), true);
 422                         taxonName.setCombinationAuthorship(team);
 423                     }
 424                     if (team == null) {
 425                         if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 426                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
 427                         }
 428                         else if (getFromMap(atomisedMap, "AuthorTeam") != null) {
 429                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeam"));
 430                         }
 431                     }
 432                     if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
 433                         team = Team.NewInstance();
 434                         team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
 435                         taxonName.setCombinationAuthorship(team);
 436                     }
 437                     if (taxonName.hasProblem()) {
 438                         logger.info("pb ICBN");
 439                         problem = true;
 440                     }
 441                     else {
 442                         return taxonName;
 443                     }
 444                 }
 445                 else if (state.getDataHolder().getNomenclatureCode().equals("Bacterial") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNB.getUuid())) {
 446                     NonViralName<BacterialName> taxonName = TaxonNameFactory.NewBacterialInstance(null);
 447                     taxonName.setFullTitleCache(fullName, true);
 448                     taxonName.setGenusOrUninomial(getFromMap(atomisedMap, "Genus"));
 449                     taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
 450                     taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "Species")));
 451                     taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "SubspeciesEpithet")));
 452
 453                     if (taxonName.getGenusOrUninomial() != null){
 454                         taxonName.setRank(Rank.GENUS());
 455                     }
 456                     else if (taxonName.getInfraGenericEpithet() != null){
 457                         taxonName.setRank(Rank.SUBGENUS());
 458                     }
 459                     else if (taxonName.getSpecificEpithet() != null){
 460                         taxonName.setRank(Rank.SPECIES());
 461                     }
 462                     else if (taxonName.getInfraSpecificEpithet() != null){
 463                         taxonName.setRank(Rank.SUBSPECIES());
 464                     }
 465
 466                     if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 467                         Team team = Team.NewInstance();
 468                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
 469                         taxonName.setCombinationAuthorship(team);
 470                     }
 471                     if (getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear") != null) {
 472                         Team team = Team.NewInstance();
 473                         team.setTitleCache(getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear"), true);
 474                         taxonName.setBasionymAuthorship(team);
 475                     }
 476                     if (taxonName.hasProblem()) {
 477                         logger.info("pb ICNB");
 478                         problem = true;
 479                     }
 480                     else {
 481                         return taxonName;
 482                     }
 483                 }
 484                 else if (state.getDataHolder().getNomenclatureCode().equals("Cultivar")) {
 485                     CultivarPlantName taxonName = TaxonNameFactory.NewCultivarInstance(null);
 486
 487                     if (taxonName.hasProblem()) {
 488                         logger.info("pb ICNCP");
 489                         problem = true;
 490                     }
 491                     else {
 492                         return taxonName;
 493                     }
 494                     return taxonName;
 495                 }
 496
 497                 if (problem) {
 498                     logger.info("Problem im setTaxonNameByType ");
 499                     TaxonNameBase<?,?> taxonName = TaxonNameFactory.NewNonViralInstance(null);
 500                     taxonName.setFullTitleCache(fullName, true);
 501                     return taxonName;
 502                 }
 503                 TaxonNameBase<?,?> tn = TaxonNameFactory.NewNonViralInstance(null);
 504                 return tn;
 505             }
 506
 507             /**
 508              * Get a formated string from a hashmap
 509              * @param atomisedMap
 510              * @param key
 511              * @return
 512              */
 513             private String getFromMap(HashMap<String, String> atomisedMap, String key) {
 514                 String value = null;
 515                 if (atomisedMap.containsKey(key)) {
 516                     value = atomisedMap.get(key);
 517                 }
 518
 519                 try {
 520                     if (value != null && key.matches(".*Year.*")) {
 521                         value = value.trim();
 522                         if (value.matches("[a-z A-Z ]*[0-9]{4}$")) {
 523                             String tmp = value.split("[0-9]{4}$")[0];
 524                             int year = Integer.parseInt(value.split(tmp)[1]);
 525                             if (year >= 1752) {
 526                                 value = tmp;
 527                             }
 528                             else {
 529                                 value = null;
 530                             }
 531                         }
 532                         else {
 533                             value = null;
 534                         }
 535                     }
 536                 }
 537                 catch (Exception e) {
 538                     value = null;
 539                 }
 540                 return value;
 541             }
 542
 543             /**
 544              * Very fast and dirty implementation to allow handling of transient objects as described in
 545              * https://dev.e-taxonomy.eu/trac/ticket/3726
 546              *
 547              * Not yet complete.
 548              *
 549              * @param cdmBase
 550              * @param state
 551              */
 552             protected UUID save(CdmBase cdmBase, SpecimenImportStateBase state) {
 553                 ICdmRepository cdmRepository = state.getConfig().getCdmAppController();
 554                 if (cdmRepository == null){
 555                     cdmRepository = this;
 556                 }
 557
 558                 if (cdmBase.isInstanceOf(LanguageString.class)){
 559                     return cdmRepository.getTermService().saveLanguageData(CdmBase.deproxy(cdmBase, LanguageString.class));
 560                 }else if (cdmBase.isInstanceOf(SpecimenOrObservationBase.class)){
 561                     return cdmRepository.getOccurrenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, SpecimenOrObservationBase.class));
 562                 }else if (cdmBase.isInstanceOf(Reference.class)){
 563                     return cdmRepository.getReferenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, Reference.class));
 564                 }else if (cdmBase.isInstanceOf(Classification.class)){
 565                     return cdmRepository.getClassificationService().saveOrUpdate(CdmBase.deproxy(cdmBase, Classification.class));
 566                 }else if (cdmBase.isInstanceOf(AgentBase.class)){
 567                     return cdmRepository.getAgentService().saveOrUpdate(CdmBase.deproxy(cdmBase, AgentBase.class));
 568                 }else if (cdmBase.isInstanceOf(Collection.class)){
 569                     return cdmRepository.getCollectionService().saveOrUpdate(CdmBase.deproxy(cdmBase, Collection.class));
 570                 }else if (cdmBase.isInstanceOf(DescriptionBase.class)){
 571                     return cdmRepository.getDescriptionService().saveOrUpdate(CdmBase.deproxy(cdmBase, DescriptionBase.class));
 572                 }else if (cdmBase.isInstanceOf(TaxonBase.class)){
 573                     return cdmRepository.getTaxonService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonBase.class));
 574                 }else if (cdmBase.isInstanceOf(TaxonNameBase.class)){
 575                     return cdmRepository.getNameService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonNameBase.class));
 576                 }else if (cdmBase.isInstanceOf(TaxonNode.class)){
 577                 return cdmRepository.getTaxonNodeService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonNode.class));
 578             }else{
 579                     throw new IllegalArgumentException("Class not supported in save method: " + CdmBase.deproxy(cdmBase, CdmBase.class).getClass().getSimpleName());
 580                 }
 581
 582             }
 583
 584
 585             protected SpecimenOrObservationBase findExistingSpecimen(String unitId, SpecimenImportStateBase state){
 586                 ICdmRepository cdmAppController = state.getConfig().getCdmAppController();
 587                 if(cdmAppController==null){
 588                     cdmAppController = this;
 589                 }
 590                 FindOccurrencesConfigurator config = new FindOccurrencesConfigurator();
 591                 config.setSignificantIdentifier(unitId);
 592                 Pager<SpecimenOrObservationBase> existingSpecimens = cdmAppController.getOccurrenceService().findByTitle(config);
 593                 if(!existingSpecimens.getRecords().isEmpty()){
 594                     if(existingSpecimens.getRecords().size()==1){
 595                         return existingSpecimens.getRecords().iterator().next();
 596                     }
 597                 }
 598                 return null;
 599             }
 600
 601             protected abstract void importAssociatedUnits(STATE state, Object item, DerivedUnitFacade derivedUnitFacade);
 602
 603             /**
 604              * getFacade : get the DerivedUnitFacade based on the recordBasis
 605              * @param state
 606              *
 607              * @return DerivedUnitFacade
 608              */
 609             protected DerivedUnitFacade getFacade(STATE state) {
 610                 if(DEBUG) {
 611                     logger.info("getFacade()");
 612                 }
 613                 SpecimenOrObservationType type = null;
 614
 615                 // create specimen
 616                 if (NB((state.getDataHolder().getRecordBasis())) != null) {
 617                     if (state.getDataHolder().getRecordBasis().toLowerCase().startsWith("s") || state.getDataHolder().getRecordBasis().toLowerCase().indexOf("specimen")>-1) {// specimen
 618                         type = SpecimenOrObservationType.PreservedSpecimen;
 619                     }
 620                     if (state.getDataHolder().getRecordBasis().toLowerCase().startsWith("o") ||state.getDataHolder().getRecordBasis().toLowerCase().indexOf("observation")>-1 ) {
 621                         type = SpecimenOrObservationType.Observation;
 622                     }
 623                     if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("fossil")>-1){
 624                         type = SpecimenOrObservationType.Fossil;
 625                     }
 626                     if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("living")>-1) {
 627                         type = SpecimenOrObservationType.LivingSpecimen;
 628                     }
 629                     if (type == null) {
 630                         logger.info("The basis of record does not seem to be known: " + state.getDataHolder().getRecordBasis());
 631                         type = SpecimenOrObservationType.DerivedUnit;
 632                     }
 633                     // TODO fossils?
 634                 } else {
 635                     logger.info("The basis of record is null");
 636                     type = SpecimenOrObservationType.DerivedUnit;
 637                 }
 638                 DerivedUnitFacade derivedUnitFacade = DerivedUnitFacade.NewInstance(type);
 639                 return derivedUnitFacade;
 640             }
 641
 642             /**
 643              * Look if the Institution does already exist
 644              * @param institutionCode: a string with the institutioncode
 645              * @param config : the configurator
 646              * @return the Institution (existing or new)
 647              */
 648             protected Institution getInstitution(String institutionCode, STATE state) {
 649                 SpecimenImportConfiguratorBase config = state.getConfig();
 650                 Institution institution=null;
 651                 institution = (Institution)state.institutions.get(institutionCode);
 652                 if (institution != null){
 653                     return institution;
 654                 }
 655                 List<Institution> institutions;
 656                 try {
 657                     institutions = getAgentService().searchInstitutionByCode(institutionCode);
 658
 659                 } catch (Exception e) {
 660                     institutions = new ArrayList<Institution>();
 661                     logger.warn(e);
 662                 }
 663                 if (institutions.size() > 0 && config.isReuseExistingMetaData()) {
 664                     for (Institution institut:institutions){
 665                         try{
 666                             if (institut.getCode().equalsIgnoreCase(institutionCode)) {
 667                                 institution=institut;
 668                                 break;
 669                             }
 670                         }catch(Exception e){logger.warn("no institution code in the db");}
 671                     }
 672                 }
 673                 if(DEBUG) {
 674                     if(institution !=null) {
 675                         logger.info("getinstitution " + institution.toString());
 676                     }
 677                 }
 678                 if (institution == null){
 679                     // create institution
 680                     institution = Institution.NewInstance();
 681                     institution.setCode(institutionCode);
 682                     institution.setTitleCache(institutionCode, true);
 683                     UUID uuid = save(institution, state);
 684                 }
 685
 686
 687                 state.institutions.put(institutionCode, institution);
 688                 return institution;
 689             }
 690
 691             /**
 692              * Look if the Collection does already exist
 693              * @param collectionCode
 694              * @param collectionCode: a string
 695              * @param config : the configurator
 696              * @return the Collection (existing or new)
 697              */
 698             protected Collection getCollection(Institution institution, String collectionCode, STATE state) {
 699                 SpecimenImportConfiguratorBase config = state.getConfig();
 700                 Collection collection = null;
 701                 List<Collection> collections;
 702                 collection = (Collection) state.collections.get(collectionCode);
 703                 if (collection != null){
 704                     return collection;
 705                 }
 706                 try {
 707                     collections = getCollectionService().searchByCode(collectionCode);
 708                 } catch (Exception e) {
 709                     collections = new ArrayList<Collection>();
 710                 }
 711                 if (collections.size() > 0 && config.isReuseExistingMetaData()) {
 712                     for (Collection coll:collections){
 713                         if (coll.getCode() != null && coll.getInstitute() != null
 714                                 && coll.getCode().equalsIgnoreCase(collectionCode) && coll.getInstitute().equals(institution)) {
 715                             collection = coll;
 716                             break;
 717                         }
 718                     }
 719                 }
 720
 721                 if(collection == null){
 722                     collection =Collection.NewInstance();
 723                     collection.setCode(collectionCode);
 724                     collection.setInstitute(institution);
 725                     collection.setTitleCache(collectionCode);
 726                     UUID uuid = save(collection, state);
 727                 }
 728
 729
 730
 731                 state.collections.put(collectionCode, collection);
 732
 733                 return collection;
 734             }
 735
 736             /**
 737              * @param reference
 738              * @param citationDetail
 739              * @return
 740              */
 741             //FIXME this method is highly critical, because
 742             //  * it will have serious performance and memory problems with large databases
 743             //        (databases may easily have >1 Mio source records)
 744             //  * it does not make sense to search for existing sources and then clone them
 745             //    we need to search for existing references instead and use them (if exist)
 746             //    for our new source.
 747             protected IdentifiableSource getIdentifiableSource(Reference reference, String citationDetail) {
 748
 749               /*  List<IdentifiableSource> issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
 750
 751
 752                 if (reference != null){
 753                     try {
 754                         for (OriginalSourceBase<?> osb: issTmp){
 755                             if (osb.getCitation() != null && osb.getCitation().getTitleCache().equalsIgnoreCase(reference.getTitleCache())){
 756                                 String osbDetail = osb.getCitationMicroReference();
 757                                 if ((StringUtils.isBlank(osbDetail) && StringUtils.isBlank(citationDetail))
 758                                         || (osbDetail != null && osbDetail.equalsIgnoreCase(citationDetail)) ) {
 759 //                                  System.out.println("REFERENCE FOUND RETURN EXISTING SOURCE");
 760                                     return (IdentifiableSource) osb.clone();
 761                                 }
 762                             }
 763                         }
 764                     } catch (CloneNotSupportedException e) {
 765                         throw new RuntimeException(e);
 766                     } catch (Exception e1){
 767                         e1.printStackTrace();
 768                     }
 769                 }
 770         */
 771                 IdentifiableSource sour = IdentifiableSource.NewInstance(OriginalSourceType.Import,null,null, reference,citationDetail);
 772                 return sour;
 773             }
 774
 775             /**
 776              * Add the hierarchy for a Taxon(add higher taxa)
 777              * @param classification
 778              * @param taxon: a taxon to add as a node
 779              * @param state: the ABCD import state
 780              */
 781             protected void addParentTaxon(Taxon taxon, STATE state, boolean preferredFlag, Classification classification){
 782                 INonViralName  nvname = taxon.getName();
 783                 Rank rank = nvname.getRank();
 784                 Taxon genus =null;
 785                 Taxon subgenus =null;
 786                 Taxon species = null;
 787                 Taxon subspecies = null;
 788                 Taxon parent = null;
 789                 if(rank!=null){
 790                     if (rank.isLower(Rank.GENUS() )){
 791                         String genusOrUninomial = nvname.getGenusOrUninomial();
 792                         TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(genusOrUninomial, Rank.GENUS(), preferredFlag, state, -1);
 793                         genus = getOrCreateTaxonForName(taxonName, state);
 794                         if (genus == null){
 795                             logger.debug("The genus should not be null " + taxonName);
 796                         }
 797                         if (preferredFlag) {
 798                             parent = linkParentChildNode(null, genus, classification, state);
 799                         }
 800
 801                     }
 802                     if (rank.isLower(Rank.SUBGENUS())){
 803                         String prefix = nvname.getGenusOrUninomial();
 804                         String name = nvname.getInfraGenericEpithet();
 805                         if (name != null){
 806                             TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(prefix+" "+name, Rank.SUBGENUS(), preferredFlag, state, -1);
 807                             subgenus = getOrCreateTaxonForName(taxonName, state);
 808                             if (preferredFlag) {
 809                                 parent = linkParentChildNode(genus, subgenus, classification, state);
 810                             }            }
 811                     }
 812                     if (rank.isLower(Rank.SPECIES())){
 813                         if (subgenus!=null){
 814                             String prefix = nvname.getGenusOrUninomial();
 815                             String name = nvname.getInfraGenericEpithet();
 816                             String spe = nvname.getSpecificEpithet();
 817                             if (spe != null){
 818                                 TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(prefix+" "+name+" "+spe, Rank.SPECIES(), preferredFlag, state, -1);
 819                                 species = getOrCreateTaxonForName(taxonName, state);
 820                                 if (preferredFlag) {
 821                                     parent = linkParentChildNode(subgenus, species, classification, state);
 822                                 }
 823                             }
 824                         }
 825                         else{
 826                             String prefix = nvname.getGenusOrUninomial();
 827                             String name = nvname.getSpecificEpithet();
 828                             if (name != null){
 829                                 TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(prefix+" "+name, Rank.SPECIES(), preferredFlag, state, -1);
 830                                 species = getOrCreateTaxonForName(taxonName, state);
 831                                 if (preferredFlag) {
 832                                     parent = linkParentChildNode(genus, species, classification, state);
 833                                 }
 834                             }
 835                         }
 836                     }
 837                     if (rank.isLower(Rank.INFRASPECIES())){
 838                         TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(nvname.getFullTitleCache(), Rank.SUBSPECIES(), preferredFlag, state, -1);
 839                         subspecies = getOrCreateTaxonForName(taxonName, state);
 840                         if (preferredFlag) {
 841                             parent = linkParentChildNode(species, subspecies, classification, state);
 842                         }
 843                     }
 844                 }
 845                 if (preferredFlag && parent!=taxon ) {
 846                     linkParentChildNode(parent, taxon, classification, state);
 847                 }
 848             }
 849
 850             /**
 851              * Link a parent to a child and save it in the current classification
 852              * @param parent: the higher Taxon
 853              * @param child : the lower (or current) Taxon
 854              * return the Taxon from the new created Node
 855              * @param classification
 856              * @param state
 857              */
 858             protected Taxon linkParentChildNode(Taxon parent, Taxon child, Classification classification, STATE state) {
 859                 TaxonNode node =null;
 860                 if (parent != null) {
 861                     parent = (Taxon) getTaxonService().find(parent.getUuid());
 862                     child = (Taxon) getTaxonService().find(child.getUuid());
 863                     //here we do not have to check if the taxon nodes already exists
 864                     //this is done by classification.addParentChild()
 865                     //do not add child node if it already exists
 866                     if(hasTaxonNodeInClassification(child, classification)){
 867                         return child;
 868                     }
 869                     else{
 870                         node = classification.addParentChild(parent, child, state.getRef(), "");
 871                         save(node, state);
 872                     }
 873                 }
 874                 else {
 875                     if (child == null){
 876                         logger.debug("The child should not be null!");
 877                     }
 878                     child = (Taxon) getTaxonService().find(child.getUuid());
 879                     //do not add child node if it already exists
 880                     if(hasTaxonNodeInClassification(child, classification)){
 881                         return child;
 882                     }
 883                     else{
 884                         node = classification.addChildTaxon(child, state.getRef(), null);
 885                         save(node, state);
 886                     }
 887                 }
 888                 if(node!=null){
 889                     state.getReport().addTaxonNode(node);
 890                     return node.getTaxon();
 891                 }
 892                 String message = "Could not create taxon node for " +child;
 893                 state.getReport().addInfoMessage(message);
 894                 logger.warn(message);
 895                 return null;
 896             }
 897
 898             protected Taxon getOrCreateTaxonForName(TaxonNameBase<?, ?> taxonNameBase, STATE state){
 899                 if (taxonNameBase != null){
 900                 Set<Taxon> acceptedTaxa = taxonNameBase.getTaxa();
 901                 if(acceptedTaxa.size()>0){
 902                     Taxon firstAcceptedTaxon = acceptedTaxa.iterator().next();
 903                     if(acceptedTaxa.size()>1){
 904                         String message = "More than one accepted taxon was found for taxon name: "
 905                                 + taxonNameBase.getTitleCache() + "!\n" + firstAcceptedTaxon + "was chosen for "+state.getDerivedUnitBase();
 906                         state.getReport().addInfoMessage(message);
 907                         logger.warn(message);
 908                     }
 909                     else{
 910                         return firstAcceptedTaxon;
 911                     }
 912                 }
 913                 else{
 914                     Set<TaxonBase> taxonAndSynonyms = taxonNameBase.getTaxonBases();
 915                     for (TaxonBase taxonBase : taxonAndSynonyms) {
 916                         if(taxonBase.isInstanceOf(Synonym.class)){
 917                             Synonym synonym = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
 918                             Taxon acceptedTaxonOfSynonym = synonym.getAcceptedTaxon();
 919                             if(acceptedTaxonOfSynonym == null){
 920                                 String message = "No accepted taxon could be found for taxon name: "
 921                                         + taxonNameBase.getTitleCache()
 922                                         + "!";
 923                                 state.getReport().addInfoMessage(message);
 924                                 logger.warn(message);
 925                             }
 926                             else{
 927                                 return acceptedTaxonOfSynonym;
 928                             }
 929                         }
 930                     }
 931                 }
 932                 Taxon taxon = Taxon.NewInstance(taxonNameBase, state.getRef());
 933                 save(taxon, state);
 934                 state.getReport().addTaxon(taxon);
 935                 logger.info("Created new taxon "+ taxon);
 936                 return taxon;
 937                 }
 938                 return null;
 939
 940             }
 941
 942             private boolean hasTaxonNodeInClassification(Taxon taxon, Classification classification){
 943                 if(taxon.getTaxonNodes()!=null){
 944                     for (TaxonNode node : taxon.getTaxonNodes()){
 945                         if(node.getClassification().equals(classification)){
 946                             return true;
 947                         }
 948                     }
 949                 }
 950                 return false;
 951             }
 952
 953             /**
 954              * HandleIdentifications : get the scientific names present in the ABCD
 955              * document and store link them with the observation/specimen data
 956              * @param state: the current ABCD import state
 957              * @param derivedUnitFacade : the current derivedunitfacade
 958              */
 959             protected void handleIdentifications(STATE state, DerivedUnitFacade derivedUnitFacade) {
 960                SpecimenImportConfiguratorBase config = state.getConfig();
 961
 962
 963                 String scientificName = "";
 964                 boolean preferredFlag = false;
 965
 966                 if (state.getDataHolder().getNomenclatureCode() == ""){
 967                     if (config.getNomenclaturalCode() != null){
 968                         if (config.getNomenclaturalCode() != null){
 969                             state.getDataHolder().setNomenclatureCode(config.getNomenclaturalCode().toString());
 970                         }
 971                     }
 972                 }
 973
 974                 for (int i = 0; i < state.getDataHolder().getIdentificationList().size(); i++) {
 975                     Identification identification = state.getDataHolder().getIdentificationList().get(i);
 976                     scientificName = identification.getScientificName().replaceAll(" et ", " & ");
 977
 978                     String preferred = identification.getPreferred();
 979                     preferredFlag = false;
 980                     if (preferred != null){
 981                     if (preferred.equals("1") || preferred.toLowerCase().indexOf("true") != -1 || state.getDataHolder().getIdentificationList().size()==1) {
 982                         preferredFlag = true;
 983                     }
 984
 985                     }
 986                     if (identification.getCode() != null){
 987                     if (identification.getCode().indexOf(':') != -1) {
 988                         state.getDataHolder().setNomenclatureCode(identification.getCode().split(COLON)[1]);
 989                     }
 990                     else{
 991                         state.getDataHolder().setNomenclatureCode(identification.getCode());
 992                     }
 993                     }
 994                     TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(scientificName, null, preferredFlag, state, i);
 995                     Taxon taxon = getOrCreateTaxonForName(taxonName, state);
 996                     addTaxonNode(taxon, state,preferredFlag);
 997                     linkDeterminationEvent(state, taxon, preferredFlag, derivedUnitFacade, identification.getIdentifier(), identification.getDate());
 998                 }
 999             }
1000
1001             /**
1002              * @param taxon : a taxon to add as a node
1003              * @param state : the ABCD import state
1004              */
1005             protected void addTaxonNode(Taxon taxon, STATE state, boolean preferredFlag) {
1006                 SpecimenImportConfiguratorBase config = state.getConfig();
1007                 logger.info("link taxon to a taxonNode "+taxon.getTitleCache());
1008                 //only add nodes if not already existing in current classification or default classification
1009
1010                 //check if node exists in current classification
1011                 //NOTE: we cannot use hasTaxonNodeInClassification() here because we are first creating it here
1012                 if (!existsInClassification(taxon,state.getClassification(), state)){
1013                     if(config.isMoveNewTaxaToDefaultClassification()){
1014                         //check if node exists in default classification
1015                         if (!existsInClassification(taxon, state.getDefaultClassification(), state)){
1016                             addParentTaxon(taxon, state, preferredFlag, state.getDefaultClassification());
1017                         }
1018                     }
1019                     else {
1020                         //add non-existing taxon to current classification
1021                         addParentTaxon(taxon, state, preferredFlag, state.getClassification());
1022                     }
1023                 }
1024             }
1025
1026
1027             private boolean existsInClassification(Taxon taxon, Classification classification, STATE state){
1028                 boolean exist = false;
1029                 ICdmRepository cdmAppController = state.getConfig().getCdmAppController();
1030             if(cdmAppController==null){
1031                 cdmAppController = this;
1032             }
1033             if (classification != null){
1034                 if (!taxon.getTaxonNodes().isEmpty()){
1035                     for (TaxonNode node:taxon.getTaxonNodes()){
1036                         if (node.getClassification().equals(classification)){
1037                             return true;
1038                         }
1039                     }
1040                 }
1041 // we do not need this because we already searched for taxa in db in the previous steps
1042 //              List<UuidAndTitleCache<TaxonNode>> uuidAndTitleCacheOfAllTaxa = cdmAppController.getClassificationService().getTaxonNodeUuidAndTitleCacheOfAcceptedTaxaByClassification(classification.getUuid());
1043 //              if (uuidAndTitleCacheOfAllTaxa != null){
1044 //                      for (UuidAndTitleCache p : uuidAndTitleCacheOfAllTaxa){
1045 //                          try{
1046 //                              if(p.getTitleCache().equals(taxon.getTitleCache())) {
1047 //                                  exist = true;
1048 //                              }
1049 //                          }
1050 //                          catch(Exception e){
1051 //                              logger.warn("TaxonNode doesn't seem to have a taxon");
1052 //                          }
1053 //                      }
1054 //              }
1055             }
1056                 return exist;
1057             }
1058
1059             /**
1060              * join DeterminationEvent to the Taxon Object
1061              * @param state : the ABCD import state
1062              * @param taxon: the current Taxon
1063              * @param preferredFlag :if the current name is preferred
1064              * @param derivedFacade : the derived Unit Facade
1065              */
1066             @SuppressWarnings("rawtypes")
1067         protected void linkDeterminationEvent(STATE state, Taxon taxon, boolean preferredFlag,  DerivedUnitFacade derivedFacade, String identifierStr, String dateStr) {
1068                 SpecimenImportConfiguratorBase config = state.getConfig();
1069                 if(DEBUG){
1070                     logger.info("start linkdetermination with taxon:" + taxon.getUuid()+", "+taxon);
1071                 }
1072
1073                 DeterminationEvent determinationEvent = DeterminationEvent.NewInstance();
1074                 determinationEvent.setTaxonName(taxon.getName());
1075                 determinationEvent.setPreferredFlag(preferredFlag);
1076
1077
1078                 determinationEvent.setIdentifiedUnit(state.getDerivedUnitBase());
1079                 if (state.getPersonStore().get(identifierStr) != null){
1080                     determinationEvent.setActor((AgentBase)state.getPersonStore().get(identifierStr));
1081                 } else if (identifierStr != null){
1082                     Person identifier = Person.NewTitledInstance(identifierStr);
1083                     determinationEvent.setActor(identifier);
1084                 }
1085                 if (dateStr != null){
1086                     determinationEvent.setTimeperiod(TimePeriodParser.parseString(dateStr));
1087                 }
1088                 state.getDerivedUnitBase().addDetermination(determinationEvent);
1089
1090                 if(DEBUG){
1091                     logger.info("NB TYPES INFO: "+ state.getDataHolder().getStatusList().size());
1092                 }
1093                 for (SpecimenTypeDesignationStatus specimenTypeDesignationstatus : state.getDataHolder().getStatusList()) {
1094                     if (specimenTypeDesignationstatus != null) {
1095                         if(DEBUG){
1096                             logger.info("specimenTypeDesignationstatus :"+ specimenTypeDesignationstatus);
1097                         }
1098
1099                         ICdmRepository cdmAppController = config.getCdmAppController();
1100                         if(cdmAppController == null){
1101                             cdmAppController = this;
1102                         }
1103                         specimenTypeDesignationstatus = (SpecimenTypeDesignationStatus) cdmAppController.getTermService().find(specimenTypeDesignationstatus.getUuid());
1104                         //Designation
1105                         TaxonNameBase<?,?> name = taxon.getName();
1106                         SpecimenTypeDesignation designation = SpecimenTypeDesignation.NewInstance();
1107
1108                         designation.setTypeStatus(specimenTypeDesignationstatus);
1109                         designation.setTypeSpecimen(state.getDerivedUnitBase());
1110                         name.addTypeDesignation(designation, true);
1111                     }
1112                 }
1113
1114                 for (String[] fullReference : state.getDataHolder().getReferenceList()) {
1115
1116
1117                     String strReference=fullReference[0];
1118                     String citationDetail = fullReference[1];
1119                     String citationURL = fullReference[2];
1120                     List<Reference> references = getReferenceService().listByTitle(Reference.class, "strReference", MatchMode.EXACT, null, null, null, null, null);
1121
1122                     if (!references.isEmpty()){
1123                         Reference reference = null;
1124                         for (Reference refe: references) {
1125                             if (refe.getTitleCache().equalsIgnoreCase(strReference)) {
1126                                 reference =refe;
1127                                 break;
1128                             }
1129                         }
1130                         if (reference ==null){
1131                             reference = ReferenceFactory.newGeneric();
1132                             reference.setTitleCache(strReference, true);
1133                             save(reference, state);
1134                         }
1135                         determinationEvent.addReference(reference);
1136                     }
1137                 }
1138                 save(state.getDerivedUnitBase(), state);
1139
1140                 if (config.isAddIndividualsAssociationsSuchAsSpecimenAndObservations() && preferredFlag) {
1141                     //do not add IndividualsAssociation to non-preferred taxa
1142                     if(DEBUG){
1143                         logger.info("isDoCreateIndividualsAssociations");
1144                     }
1145
1146                     makeIndividualsAssociation(state, taxon, determinationEvent);
1147
1148                     save(state.getDerivedUnitBase(), state);
1149                 }
1150             }
1151
1152             /**
1153              * create and link each association (specimen, observation..) to the accepted taxon
1154              * @param state : the ABCD import state
1155              * @param taxon: the current Taxon
1156              * @param determinationEvent:the determinationevent
1157              */
1158             protected void makeIndividualsAssociation(STATE state, Taxon taxon, DeterminationEvent determinationEvent) {
1159                 SpecimenImportConfiguratorBase config = state.getConfig();
1160                 SpecimenUserInteraction sui = config.getSpecimenUserInteraction();
1161
1162                 if (DEBUG) {
1163                     logger.info("MAKE INDIVIDUALS ASSOCIATION");
1164                 }
1165
1166                 TaxonDescription taxonDescription = null;
1167                 Set<TaxonDescription> descriptions= taxon.getDescriptions();
1168 //              if (((Abcd206ImportConfigurator) state.getConfig()).isInteractWithUser()){
1169 //                  if(!state.isDescriptionGroupSet()){
1170 //                      taxonDescription = sui.askForDescriptionGroup(descriptions);
1171 //                      state.setDescriptionGroup(taxonDescription);
1172 //                      state.setDescriptionGroupSet(true);
1173 //                  }else{
1174 //                      taxonDescription=state.getDescriptionGroup();
1175 //                  }
1176 //              } else {
1177                     for (TaxonDescription description : descriptions){
1178                         Set<IdentifiableSource> sources =  new HashSet<>();
1179                         sources.addAll(description.getTaxon().getSources());
1180                         sources.addAll(description.getSources());
1181                         for (IdentifiableSource source:sources){
1182                             if(state.getRef().equals(source.getCitation())) {
1183                                 taxonDescription = description;
1184                             }
1185                         }
1186                     }
1187               //  }
1188                 if (taxonDescription == null){
1189                     taxonDescription = TaxonDescription.NewInstance(taxon, false);
1190                     if(sourceNotLinkedToElement(taxonDescription,state.getRef(),null)) {
1191                         taxonDescription.addSource(OriginalSourceType.Import, null, null, state.getRef(), null);
1192                     }
1193                     state.setDescriptionGroup(taxonDescription);
1194                     taxon.addDescription(taxonDescription);
1195                 }
1196
1197                 //PREPARE REFERENCE QUESTIONS
1198
1199                 Map<String,OriginalSourceBase<?>> sourceMap = new HashMap<String, OriginalSourceBase<?>>();
1200
1201                 List<IdentifiableSource> issTmp = new ArrayList<>();//getCommonService().list(IdentifiableSource.class, null, null, null, null);
1202                 List<DescriptionElementSource> issTmp2 = new ArrayList<>();//getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1203
1204                 Set<OriginalSourceBase> osbSet = new HashSet<OriginalSourceBase>();
1205                 if(issTmp2!=null) {
1206                     osbSet.addAll(issTmp2);
1207                 }
1208                 if(issTmp!=null) {
1209                     osbSet.addAll(issTmp);
1210                 }
1211
1212
1213                 addToSourceMap(sourceMap, osbSet);
1214
1215 //              if (((Abcd206ImportConfigurator) state.getConfig()).isInteractWithUser()){
1216 //                  List<OriginalSourceBase<?>> res = null;
1217 //                  if(!state.isDescriptionSourcesSet()){
1218 //                      res = sui.askForSource(sourceMap, "the description group ("+taxon+")",
1219 //                              "The current reference is "+state.getRef().getTitleCache(),getReferenceService(), state.getDataHolder().docSources);
1220 //                      state.setDescriptionRefs(res);
1221 //                      state.setDescriptionSourcesSet(true);
1222 //                  }
1223 //                  else{
1224 //                      res=state.getDescriptionRefs();
1225 //                  }
1226 //                  if(res !=null) {
1227 //                      for (OriginalSourceBase<?> sour:res){
1228 //                          if(sour.isInstanceOf(IdentifiableSource.class)){
1229 //                              try {
1230 //                                  if(sourceNotLinkedToElement(taxonDescription,sour)) {
1231 //                                      taxonDescription.addSource((IdentifiableSource)sour.clone());
1232 //                                  }
1233 //                              } catch (CloneNotSupportedException e) {
1234 //                                  logger.warn("no cloning?");
1235 //                              }
1236 //                          }else{
1237 //                              if(sourceNotLinkedToElement(taxonDescription,sour)) {
1238 //                                  taxonDescription.addSource(OriginalSourceType.Import,null, null, sour.getCitation(),sour.getCitationMicroReference());
1239 //                              }
1240 //                          }
1241 //                      }
1242 //                  }
1243 //              }
1244 //              else {
1245                     if(sourceNotLinkedToElement(taxonDescription,state.getRef(),null)) {
1246                         taxonDescription.addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1247                     }
1248 //              }
1249                 state.setDescriptionGroup(taxonDescription);
1250
1251                 IndividualsAssociation indAssociation = IndividualsAssociation.NewInstance();
1252                 Feature feature = makeFeature(state.getDerivedUnitBase());
1253                 indAssociation.setAssociatedSpecimenOrObservation(state.getDerivedUnitBase());
1254                 indAssociation.setFeature(feature);
1255 //
1256 //              if (((Abcd206ImportConfigurator) state.getConfig()).isInteractWithUser()){
1257 //                  sourceMap = new HashMap<String, OriginalSourceBase<?>>();
1258 //
1259 //                  issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
1260 //                  issTmp2 = getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1261 //
1262 //                  osbSet = new HashSet<OriginalSourceBase>();
1263 //                  if(issTmp2!=null) {
1264 //                      osbSet.addAll(issTmp2);
1265 //                  }
1266 //                  if(issTmp!=null) {
1267 //                      osbSet.addAll(issTmp);
1268 //                  }
1269 //
1270 //
1271 //                  addToSourceMap(sourceMap, osbSet);
1272 //
1273 //                  List<OriginalSourceBase<?>> sources =null;
1274 //                  if(!state.isAssociationSourcesSet()) {
1275 //                      sources = sui.askForSource(sourceMap,  "descriptive element (association) ",taxon.toString(),
1276 //                              getReferenceService(),state.getDataHolder().getDocSources());
1277 //                      state.setAssociationRefs(sources);
1278 //                      state.setAssociationSourcesSet(true);
1279 //                  }
1280 //                  else{
1281 //                      sources=state.getAssociationRefs();
1282 //                  }
1283 //                  if(sources !=null) {
1284 //                      for (OriginalSourceBase<?> source: sources) {
1285 //                          if(source !=null) {
1286 //                              if(source.isInstanceOf(DescriptionElementSource.class)){
1287 //                                  try {
1288 //                                      if(sourceNotLinkedToElement(indAssociation,source)) {
1289 //                                          indAssociation.addSource((DescriptionElementSource)source.clone());
1290 //                                      }
1291 //                                  } catch (CloneNotSupportedException e) {
1292 //                                      logger.warn("clone forbidden?");
1293 //                                  }
1294 //                              }else{
1295 //                                  if(sourceNotLinkedToElement(indAssociation,source)) {
1296 //                                      indAssociation.addSource(OriginalSourceType.Import,null, null, source.getCitation(),source.getCitationMicroReference());
1297 //                                  }
1298 //                                  try {
1299 //                                      if(sourceNotLinkedToElement(state.getDerivedUnitBase(), source)) {
1300 //                                          state.getDerivedUnitBase().addSource((IdentifiableSource) source.clone());
1301 //                                      }
1302 //                                  } catch (CloneNotSupportedException e) {
1303 //                                      // TODO Auto-generated catch block
1304 //                                      e.printStackTrace();
1305 //                                  }
1306 //                              }
1307 //
1308 //                          }
1309 //                      }
1310 //                  }
1311 //              }else {
1312                     if(sourceNotLinkedToElement(indAssociation,state.getRef(),null)) {
1313                         indAssociation.addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1314                     }
1315                     if(sourceNotLinkedToElement(state.getDerivedUnitBase(), state.getRef(),null)) {
1316                         state.getDerivedUnitBase().addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1317                     }
1318                     for (Reference citation : determinationEvent.getReferences()) {
1319                         if(sourceNotLinkedToElement(indAssociation,citation,null))
1320                         {
1321                             indAssociation.addSource(DescriptionElementSource.NewInstance(OriginalSourceType.Import, null, null, citation, null));
1322                         }
1323                         if(sourceNotLinkedToElement(state.getDerivedUnitBase(), state.getRef(),null)) {
1324                             state.getDerivedUnitBase().addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1325                         }
1326                     }
1327            //     }
1328
1329                 taxonDescription.addElement(indAssociation);
1330
1331                 save(taxonDescription, state);
1332                 save(taxon, state);
1333                 state.getReport().addDerivate(state.getDerivedUnitBase(), config);
1334                 state.getReport().addIndividualAssociation(taxon, state.getDataHolder().getUnitID(), state.getDerivedUnitBase());
1335             }
1336
1337             /**
1338              * @param derivedUnitBase2
1339              * @param ref2
1340              * @param object
1341              * @return
1342              */
1343             private boolean sourceNotLinkedToElement(DerivedUnit derivedUnitBase2, Reference b, String d) {
1344                 Set<IdentifiableSource> linkedSources = derivedUnitBase2.getSources();
1345                 for (IdentifiableSource is:linkedSources){
1346                     Reference a = is.getCitation();
1347                     String c = is.getCitationMicroReference();
1348
1349                     boolean refMatch=false;
1350                     boolean microMatch=false;
1351
1352                     try{
1353                         if (a==null && b==null) {
1354                             refMatch=true;
1355                         }
1356                         if (a!=null && b!=null) {
1357                             if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1358                                 refMatch=true;
1359                             }
1360                         }
1361                     }catch(Exception e){}
1362
1363
1364                     try{
1365                         if (c==null && d==null) {
1366                             microMatch=true;
1367                         }
1368                         if(c!=null && d!=null) {
1369                             if(c.equalsIgnoreCase(d)) {
1370                                 microMatch=true;
1371                             }
1372                         }
1373                     }
1374                     catch(Exception e){}
1375
1376                     if (microMatch && refMatch) {
1377                         return false;
1378                     }
1379
1380
1381                 }
1382                 return true;
1383             }
1384
1385             private <T extends OriginalSourceBase<?>> boolean  sourceNotLinkedToElement(ISourceable<T> sourcable, Reference reference, String microReference) {
1386                 Set<T> linkedSources = sourcable.getSources();
1387                 for (T is:linkedSources){
1388                     Reference unitReference = is.getCitation();
1389                     String unitMicroReference = is.getCitationMicroReference();
1390
1391                     boolean refMatch=false;
1392                     boolean microMatch=false;
1393
1394                     try{
1395                         if (unitReference==null && reference==null) {
1396                             refMatch=true;
1397                         }
1398                         if (unitReference!=null && reference!=null) {
1399                             if (unitReference.getTitleCache().equalsIgnoreCase(reference.getTitleCache())) {
1400                                 refMatch=true;
1401                             }
1402                         }
1403                     }catch(Exception e){}
1404
1405                     try{
1406                         if (unitMicroReference==null && microReference==null) {
1407                             microMatch=true;
1408                         }
1409                         if(unitMicroReference!=null && microReference!=null) {
1410                             if(unitMicroReference.equalsIgnoreCase(microReference)) {
1411                                 microMatch=true;
1412                             }
1413                         }
1414                     }
1415                     catch(Exception e){}
1416
1417                     if (microMatch && refMatch) {
1418                         return false;
1419                     }
1420                 }
1421                 return true;
1422             }
1423
1424             /**
1425              * look for the Feature object (FieldObs, Specimen,...)
1426              * @param unit : a specimen or obersvation base
1427              * @return the corresponding Feature
1428              */
1429             private Feature makeFeature(SpecimenOrObservationBase<?> unit) {
1430                 SpecimenOrObservationType type = unit.getRecordBasis();
1431
1432
1433
1434                 if (type.isFeatureObservation()){
1435                     return Feature.OBSERVATION();
1436                 }else if (type.isFeatureSpecimen()){
1437                     return Feature.SPECIMEN();
1438                 }else if (type == SpecimenOrObservationType.DerivedUnit){
1439                     return Feature.OBSERVATION();
1440                     //            return getFeature("Specimen or observation");
1441                 }else{
1442                     String message = "Unhandled record basis '%s' for defining individuals association feature type. Use default.";
1443                     logger.warn(String.format(message, type.getMessage()));
1444                     return Feature.OBSERVATION();
1445                     //            return getFeature("Specimen or observation");
1446
1447                 }
1448             }
1449
1450
1451             /**
1452              * @param sourceMap
1453              * @param osbSet
1454              */
1455             protected void addToSourceMap(Map<String, OriginalSourceBase<?>> sourceMap, Set<OriginalSourceBase> osbSet) {
1456                 for( OriginalSourceBase<?> osb:osbSet) {
1457                     if(osb.getCitation()!=null && osb.getCitationMicroReference() !=null  && !osb.getCitationMicroReference().isEmpty()) {
1458                         try{
1459                             sourceMap.put(osb.getCitation().getTitleCache()+ "---"+osb.getCitationMicroReference(),osb);
1460                         }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1461                     } else if(osb.getCitation()!=null){
1462                         try{
1463                             sourceMap.put(osb.getCitation().getTitleCache(),osb);
1464                         }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1465                     }
1466                 }
1467             }
1468
1469 }