cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/SpecimenImportBase.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.specimen;
  11
  12
  13 import java.util.ArrayList;
  14 import java.util.HashMap;
  15 import java.util.HashSet;
  16 import java.util.List;
  17 import java.util.Map;
  18 import java.util.Set;
  19 import java.util.UUID;
  20
  21 import org.apache.log4j.Logger;
  22
  23 import eu.etaxonomy.cdm.api.application.ICdmRepository;
  24 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
  25 import eu.etaxonomy.cdm.api.service.config.FindOccurrencesConfigurator;
  26 import eu.etaxonomy.cdm.api.service.pager.Pager;
  27 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  28 import eu.etaxonomy.cdm.io.common.CdmImportBase;
  29 import eu.etaxonomy.cdm.io.common.IImportConfigurator;
  30 import eu.etaxonomy.cdm.io.specimen.abcd206.in.Identification;
  31 import eu.etaxonomy.cdm.io.specimen.abcd206.in.SpecimenImportReport;
  32 import eu.etaxonomy.cdm.model.agent.AgentBase;
  33 import eu.etaxonomy.cdm.model.agent.Institution;
  34 import eu.etaxonomy.cdm.model.agent.Person;
  35 import eu.etaxonomy.cdm.model.agent.Team;
  36 import eu.etaxonomy.cdm.model.common.CdmBase;
  37 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  38 import eu.etaxonomy.cdm.model.common.LanguageString;
  39 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  40 import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
  41 import eu.etaxonomy.cdm.model.description.Feature;
  42 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  43 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  44 import eu.etaxonomy.cdm.model.name.INonViralName;
  45 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  46 import eu.etaxonomy.cdm.model.name.Rank;
  47 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
  48 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
  49 import eu.etaxonomy.cdm.model.name.TaxonName;
  50 import eu.etaxonomy.cdm.model.name.TaxonNameFactory;
  51 import eu.etaxonomy.cdm.model.occurrence.Collection;
  52 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  53 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
  54 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
  55 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationType;
  56 import eu.etaxonomy.cdm.model.reference.ISourceable;
  57 import eu.etaxonomy.cdm.model.reference.OriginalSourceBase;
  58 import eu.etaxonomy.cdm.model.reference.OriginalSourceType;
  59 import eu.etaxonomy.cdm.model.reference.Reference;
  60 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  61 import eu.etaxonomy.cdm.model.taxon.Classification;
  62 import eu.etaxonomy.cdm.model.taxon.Synonym;
  63 import eu.etaxonomy.cdm.model.taxon.Taxon;
  64 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  65 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  66 import eu.etaxonomy.cdm.model.term.DefinedTerm;
  67 import eu.etaxonomy.cdm.persistence.query.MatchMode;
  68 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
  69 import eu.etaxonomy.cdm.strategy.parser.ParserProblem;
  70 import eu.etaxonomy.cdm.strategy.parser.TimePeriodParser;
  71
  72
  73 /**
  74  * @author p.kelbert
  75  * @since 20.10.2008
  76  */
  77 public abstract class SpecimenImportBase<CONFIG extends IImportConfigurator, STATE extends SpecimenImportStateBase>
  78         extends CdmImportBase<CONFIG, STATE> {
  79
  80     private static final long serialVersionUID = 4423065367998125678L;
  81     private static final Logger logger = Logger.getLogger(SpecimenImportBase.class);
  82
  83         protected static final UUID SPECIMEN_SCAN_TERM = UUID.fromString("acda15be-c0e2-4ea8-8783-b9b0c4ad7f03");
  84
  85         private static final String COLON = ":";
  86
  87         protected Map<String, DefinedTerm> kindOfUnitsMap;
  88
  89
  90         @Override
  91     protected abstract void doInvoke(STATE state);
  92
  93         /**
  94          * Handle a single unit
  95          * @param state
  96          * @param item
  97          */
  98         protected abstract void handleSingleUnit(STATE state, Object item) ;
  99
 100
 101
 102         protected TaxonName getOrCreateTaxonName(String scientificName, Rank rank, boolean preferredFlag, STATE state, int unitIndexInAbcdFile){
 103             TaxonName taxonName = null;
 104         SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
 105
 106         //check atomised name data for rank
 107         //new name will be created
 108         TaxonName atomisedTaxonName = null;
 109         if (rank==null && unitIndexInAbcdFile>=0 && ((state.getDataHolder().getAtomisedIdentificationList() != null && !state.getDataHolder().getAtomisedIdentificationList().isEmpty())|| state.getDataHolder().getAtomisedIdentificationList().size() > 0)) {
 110             atomisedTaxonName = setTaxonNameByType(state.getDataHolder().getAtomisedIdentificationList().get(unitIndexInAbcdFile), scientificName, state);
 111             if(atomisedTaxonName!=null){
 112                 rank = atomisedTaxonName.getRank();
 113             }
 114         }
 115         if(config.isReuseExistingTaxaWhenPossible()){
 116             TaxonName parsedName = atomisedTaxonName;
 117             if(parsedName==null){
 118
 119                 parsedName = parseScientificName(scientificName, state, state.getReport(), rank);
 120
 121             }
 122             atomisedTaxonName = parsedName;
 123             if(config.isIgnoreAuthorship() && parsedName!=null){// && preferredFlag){
 124                 // do not ignore authorship for non-preferred names because they need
 125                 // to be created for the determination history
 126                 String nameCache = TaxonName.castAndDeproxy(parsedName).getNameCache();
 127                 List<TaxonName> names = getNameService().findNamesByNameCache(nameCache, MatchMode.EXACT, null);
 128                 if (!names.isEmpty()){
 129                      taxonName = getBestMatchingName(scientificName, new ArrayList<>(names), state);
 130                 }
 131                 if (taxonName == null && !names.isEmpty()){
 132                     taxonName = names.get(0);
 133                 }
 134
 135             } else {
 136                 //search for existing names
 137                 List<TaxonName> names = getNameService().listByTitleWithRestrictions(TaxonName.class, scientificName, MatchMode.EXACT, null, null, null, null, null);
 138                 taxonName = getBestMatchingName(scientificName, names, state);
 139                 //still nothing found -> try with the atomised name full title cache
 140                 if(taxonName==null && atomisedTaxonName!=null){
 141                     names = getNameService().listByTitleWithRestrictions(TaxonName.class, atomisedTaxonName.getFullTitleCache(), MatchMode.EXACT, null, null, null, null, null);
 142                     taxonName = getBestMatchingName(atomisedTaxonName.getTitleCache(), names, state);
 143                     //still nothing found -> try with the atomised name title cache
 144                     if(taxonName==null){
 145                         names = getNameService().listByTitleWithRestrictions(TaxonName.class, atomisedTaxonName.getTitleCache(), MatchMode.EXACT, null, null, null, null, null);
 146                         taxonName = getBestMatchingName(atomisedTaxonName.getTitleCache(), names, state);
 147                     }
 148                 }
 149
 150             }
 151
 152         }
 153
 154         if(taxonName==null && atomisedTaxonName!=null){
 155             taxonName = atomisedTaxonName;
 156             state.getReport().addName(taxonName);
 157             logger.info("Created new taxon name "+taxonName);
 158             if(taxonName.hasProblem()){
 159                 state.getReport().addInfoMessage(String.format("Created %s with parsing problems", taxonName));
 160             }
 161             if(!atomisedTaxonName.getTitleCache().equals(scientificName)){
 162                 state.getReport().addInfoMessage(String.format("Taxon %s was parsed as %s", scientificName, atomisedTaxonName.getTitleCache()));
 163             }
 164         }
 165         else if(taxonName==null){
 166             //create new taxon name
 167
 168             if (state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNAFP)){
 169                 taxonName = TaxonNameFactory.NewBotanicalInstance(rank);
 170             }else if (state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICZN)){
 171                 taxonName = TaxonNameFactory.NewZoologicalInstance(rank);
 172             }else{
 173                 taxonName = TaxonNameFactory.NewNonViralInstance(rank);
 174             }
 175             taxonName.setFullTitleCache(scientificName,true);
 176             taxonName.setTitleCache(scientificName, true);
 177             state.getReport().addName(taxonName);
 178             logger.info("Created new taxon name "+taxonName);
 179         }
 180         save(taxonName, state);
 181         return taxonName;
 182     }
 183
 184          protected TaxonName getBestMatchingName(String scientificName, java.util.Collection<TaxonName> names, STATE state){
 185                 Set<TaxonName> namesWithAcceptedTaxa = new HashSet<>();
 186                 List<TaxonName> namesWithAcceptedTaxaInClassification = new ArrayList<>();
 187                 for (TaxonName name : names) {
 188                     if(!name.getTaxa().isEmpty()){
 189                         Set<Taxon> taxa = name.getTaxa();
 190                         for (Taxon taxon:taxa){
 191                             if (!taxon.getTaxonNodes().isEmpty()){
 192                                 //use only taxa included in a classification
 193                                 for (TaxonNode node:taxon.getTaxonNodes()){
 194                                     if (state.getClassification() != null && node.getClassification().equals(state.getClassification())){
 195                                         namesWithAcceptedTaxaInClassification.add(name);
 196                                     }else {
 197                                         namesWithAcceptedTaxa.add(name);
 198                                     }
 199                                 }
 200
 201                             }
 202                         }
 203
 204                     }
 205                 }
 206                 String message = String.format("More than one taxon name was found for %s, maybe in other classifications!", scientificName);
 207                 //check for names with accepted taxa in classification
 208                 if(namesWithAcceptedTaxaInClassification.size()>0){
 209                 if(namesWithAcceptedTaxaInClassification.size()>1){
 210
 211                     state.getReport().addInfoMessage(message);
 212                     logger.warn(message);
 213                     return null;
 214                 }
 215                 return namesWithAcceptedTaxaInClassification.iterator().next();
 216             }
 217               //check for any names with accepted taxa
 218                 if(namesWithAcceptedTaxa.size()>0){
 219                     if(namesWithAcceptedTaxa.size()>1){
 220
 221                         state.getReport().addInfoMessage(message);
 222                         logger.warn(message);
 223                         return null;
 224                     }
 225                     return namesWithAcceptedTaxa.iterator().next();
 226                 }
 227                 //no names with accepted taxa found -> check accepted taxa of synonyms
 228                 List<Taxon> taxaFromSynonyms = new ArrayList<>();
 229                 for (TaxonName name : names) {
 230                     Set<TaxonBase> taxonBases = name.getTaxonBases();
 231                     for (TaxonBase taxonBase : taxonBases) {
 232                         if(taxonBase.isInstanceOf(Synonym.class)){
 233                             Synonym synonym = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
 234                             taxaFromSynonyms.add(synonym.getAcceptedTaxon());
 235                         }
 236                     }
 237                 }
 238                 if(taxaFromSynonyms.size()>0){
 239                     if(taxaFromSynonyms.size()>1){
 240                         state.getReport().addInfoMessage(message);
 241                         logger.warn(message);
 242                         return null;
 243                     }
 244                     return taxaFromSynonyms.iterator().next().getName();
 245                 }
 246                 //no accepted and no synonyms -> return one of the names and create a new taxon
 247                 if (names.isEmpty()){
 248                     return null;
 249                 }else{
 250                     return names.iterator().next();
 251                 }
 252             }
 253          /**
 254              * Parse automatically the scientific name
 255              * @param scientificName the scientific name to parse
 256              * @param state the current import state
 257              * @param report the import report
 258              * @return a parsed name
 259              */
 260
 261             protected TaxonName parseScientificName(String scientificName, STATE state, SpecimenImportReport report, Rank rank) {
 262
 263                 NonViralNameParserImpl nvnpi = NonViralNameParserImpl.NewInstance();
 264                 TaxonName taxonName = null;
 265                 boolean problem = false;
 266
 267                 if (logger.isDebugEnabled()){
 268                     logger.debug("parseScientificName " + state.getDataHolder().getNomenclatureCode().toString());
 269                 }
 270
 271                 if (state.getDataHolder().getNomenclatureCode().toString().equals("Zoological") || state.getDataHolder().getNomenclatureCode().toString().contains("ICZN")) {
 272                     taxonName = (TaxonName)nvnpi.parseFullName(scientificName, NomenclaturalCode.ICZN, rank);
 273                     if (taxonName.hasProblem()) {
 274                         problem = true;
 275                     }
 276                 }
 277                 else if (state.getDataHolder().getNomenclatureCode().toString().equals("Botanical") || state.getDataHolder().getNomenclatureCode().toString().contains("ICBN")  || state.getDataHolder().getNomenclatureCode().toString().contains("ICNAFP")) {
 278                     taxonName = (TaxonName)nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNAFP, rank);
 279                     if (taxonName.hasProblem()) {
 280                         problem = true;
 281                     }
 282                 }
 283                 else if (state.getDataHolder().getNomenclatureCode().toString().equals("Bacterial") || state.getDataHolder().getNomenclatureCode().toString().contains("ICBN")) {
 284                     taxonName = (TaxonName)nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNB, rank);
 285                     if (taxonName.hasProblem()) {
 286                         problem = true;
 287                     }
 288                 }
 289                 else if (state.getDataHolder().getNomenclatureCode().toString().equals("Cultivar") || state.getDataHolder().getNomenclatureCode().toString().contains("ICNCP")) {
 290                     taxonName = (TaxonName)nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNCP, rank);
 291                     if (taxonName.hasProblem()) {
 292                         problem = true;
 293                     }
 294                 }
 295                 if (problem) {
 296                     String message = String.format("Parsing problems for %s", scientificName);
 297                     if(taxonName!=null){
 298                         for (ParserProblem parserProblem : taxonName.getParsingProblems()) {
 299                             message += "\n\t- "+parserProblem;
 300                         }
 301                     }
 302                     report.addInfoMessage(message);
 303                     logger.info(message);
 304                 }
 305                 return taxonName;
 306
 307             }
 308
 309             /**
 310              * Create the name without automatic parsing, either because it failed, or because the user deactivated it.
 311              * The name is built upon the ABCD fields
 312              * @param atomisedMap : the ABCD atomised fields
 313              * @param fullName : the full scientific name
 314              * @param state
 315              * @return the corresponding Botanical or Zoological or... name
 316              */
 317             protected TaxonName setTaxonNameByType(
 318                     HashMap<String, String> atomisedMap, String fullName, STATE state) {
 319                 boolean problem = false;
 320                 if (logger.isDebugEnabled()){
 321                     logger.debug("settaxonnamebytype " + state.getDataHolder().getNomenclatureCode().toString());
 322                 }
 323
 324                 if (state.getDataHolder().getNomenclatureCode().equals("Zoological") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICZN.getUuid())) {
 325                     TaxonName taxonName = TaxonNameFactory.NewZoologicalInstance(null);
 326                     taxonName.setFullTitleCache(fullName, true);
 327                     taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
 328                     taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
 329                     taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap,"SpeciesEpithet")));
 330                     taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap,"SubspeciesEpithet")));
 331
 332                     if (taxonName.getGenusOrUninomial() != null){
 333                         taxonName.setRank(Rank.GENUS());
 334                     }
 335
 336                     if (taxonName.getInfraGenericEpithet() != null){
 337                         taxonName.setRank(Rank.SUBGENUS());
 338                     }
 339
 340                     if (taxonName.getSpecificEpithet() != null){
 341                         taxonName.setRank(Rank.SPECIES());
 342                     }
 343
 344                     if (taxonName.getInfraSpecificEpithet() != null){
 345                         taxonName.setRank(Rank.SUBSPECIES());
 346                     }
 347
 348                     Team team = null;
 349                     if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 350                         team = Team.NewInstance();
 351                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
 352                     }
 353                     else {
 354                         if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 355                             team = Team.NewInstance();
 356                             team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
 357                         }
 358                     }
 359                     if (team != null) {
 360                         taxonName.setBasionymAuthorship(team);
 361                     }
 362                     else {
 363                         if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 364                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
 365                         }
 366                         else if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 367                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamAndYear"));
 368                         }
 369                     }
 370                     if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
 371                         team = Team.NewInstance();
 372                         team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
 373                         taxonName.setCombinationAuthorship(team);
 374                     }
 375                     if (taxonName.hasProblem()) {
 376                         logger.info("pb ICZN");
 377                         problem = true;
 378                     }
 379                     else {
 380                         return taxonName;
 381                     }
 382                 }
 383                 else if (state.getDataHolder().getNomenclatureCode().equals("Botanical") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNAFP.getUuid())) {
 384                     TaxonName taxonName = parseScientificName(fullName, state, state.getReport(), null);
 385                     if (taxonName != null){
 386                         return taxonName;
 387                     }
 388                     else{
 389                         taxonName = TaxonNameFactory.NewBotanicalInstance(null);
 390                     }
 391                     taxonName.setFullTitleCache(fullName, true);
 392                     taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
 393                     taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "FirstEpithet")));
 394                     taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "InfraSpeEpithet")));
 395                     try {
 396                         taxonName.setRank(Rank.getRankByName(getFromMap(atomisedMap, "Rank")));
 397                     } catch (Exception e) {
 398                         if (taxonName.getInfraSpecificEpithet() != null){
 399                             taxonName.setRank(Rank.SUBSPECIES());
 400                         }
 401                         else if (taxonName.getSpecificEpithet() != null){
 402                             taxonName.setRank(Rank.SPECIES());
 403                         }
 404                         else if (taxonName.getInfraGenericEpithet() != null){
 405                             taxonName.setRank(Rank.SUBGENUS());
 406                         }
 407                         else if (taxonName.getGenusOrUninomial() != null){
 408                             taxonName.setRank(Rank.GENUS());
 409                         }
 410                     }
 411                     Team team = null;
 412                     if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 413                         team = Team.NewInstance();
 414                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
 415                         taxonName.setBasionymAuthorship(team);
 416                     }
 417                     if (getFromMap(atomisedMap, "AuthorTeam") != null) {
 418                         team = Team.NewInstance();
 419                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeam"), true);
 420                         taxonName.setCombinationAuthorship(team);
 421                     }
 422                     if (team == null) {
 423                         if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 424                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
 425                         }
 426                         else if (getFromMap(atomisedMap, "AuthorTeam") != null) {
 427                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeam"));
 428                         }
 429                     }
 430                     if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
 431                         team = Team.NewInstance();
 432                         team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
 433                         taxonName.setCombinationAuthorship(team);
 434                     }
 435                     if (taxonName.hasProblem()) {
 436                         logger.info("pb ICBN");
 437                         problem = true;
 438                     }
 439                     else {
 440                         return taxonName;
 441                     }
 442                 }
 443                 else if (state.getDataHolder().getNomenclatureCode().equals("Bacterial") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNB.getUuid())) {
 444                     TaxonName taxonName = TaxonNameFactory.NewBacterialInstance(null);
 445                     taxonName.setFullTitleCache(fullName, true);
 446                     taxonName.setGenusOrUninomial(getFromMap(atomisedMap, "Genus"));
 447                     taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
 448                     taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "Species")));
 449                     taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "SubspeciesEpithet")));
 450
 451                     if (taxonName.getGenusOrUninomial() != null){
 452                         taxonName.setRank(Rank.GENUS());
 453                     }
 454                     else if (taxonName.getInfraGenericEpithet() != null){
 455                         taxonName.setRank(Rank.SUBGENUS());
 456                     }
 457                     else if (taxonName.getSpecificEpithet() != null){
 458                         taxonName.setRank(Rank.SPECIES());
 459                     }
 460                     else if (taxonName.getInfraSpecificEpithet() != null){
 461                         taxonName.setRank(Rank.SUBSPECIES());
 462                     }
 463
 464                     if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 465                         Team team = Team.NewInstance();
 466                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
 467                         taxonName.setCombinationAuthorship(team);
 468                     }
 469                     if (getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear") != null) {
 470                         Team team = Team.NewInstance();
 471                         team.setTitleCache(getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear"), true);
 472                         taxonName.setBasionymAuthorship(team);
 473                     }
 474                     if (taxonName.hasProblem()) {
 475                         logger.info("pb ICNB");
 476                         problem = true;
 477                     }
 478                     else {
 479                         return taxonName;
 480                     }
 481                 }
 482                 else if (state.getDataHolder().getNomenclatureCode().equals("Cultivar")) {
 483                     TaxonName taxonName = TaxonNameFactory.NewCultivarInstance(null);
 484
 485                     if (taxonName.hasProblem()) {
 486                         logger.info("pb ICNCP");
 487                         problem = true;
 488                     }
 489                     else {
 490                         return taxonName;
 491                     }
 492                     return taxonName;
 493                 }
 494
 495                 if (problem) {
 496                     logger.info("Problem im setTaxonNameByType ");
 497                     TaxonName taxonName = TaxonNameFactory.NewNonViralInstance(null);
 498                     taxonName.setFullTitleCache(fullName, true);
 499                     return taxonName;
 500                 }
 501                 TaxonName tn = TaxonNameFactory.NewNonViralInstance(null);
 502                 return tn;
 503             }
 504
 505             /**
 506              * Get a formated string from a hashmap
 507              * @param atomisedMap
 508              * @param key
 509              * @return
 510              */
 511             private String getFromMap(HashMap<String, String> atomisedMap, String key) {
 512                 String value = null;
 513                 if (atomisedMap.containsKey(key)) {
 514                     value = atomisedMap.get(key);
 515                 }
 516
 517                 try {
 518                     if (value != null && key.matches(".*Year.*")) {
 519                         value = value.trim();
 520                         if (value.matches("[a-z A-Z ]*[0-9]{4}$")) {
 521                             String tmp = value.split("[0-9]{4}$")[0];
 522                             int year = Integer.parseInt(value.split(tmp)[1]);
 523                             if (year >= 1752) {
 524                                 value = tmp;
 525                             }
 526                             else {
 527                                 value = null;
 528                             }
 529                         }
 530                         else {
 531                             value = null;
 532                         }
 533                     }
 534                 }
 535                 catch (Exception e) {
 536                     value = null;
 537                 }
 538                 return value;
 539             }
 540
 541             /**
 542              * Very fast and dirty implementation to allow handling of transient objects as described in
 543              * https://dev.e-taxonomy.eu/trac/ticket/3726
 544              *
 545              * Not yet complete.
 546              *
 547              * @param cdmBase
 548              * @param state
 549              */
 550             protected UUID save(CdmBase cdmBase, SpecimenImportStateBase state) {
 551                 ICdmRepository cdmRepository = state.getConfig().getCdmAppController();
 552                 if (cdmRepository == null){
 553                     cdmRepository = this;
 554                 }
 555
 556                 if (cdmBase.isInstanceOf(LanguageString.class)){
 557                     return cdmRepository.getTermService().saveLanguageData(CdmBase.deproxy(cdmBase, LanguageString.class));
 558                 }else if (cdmBase.isInstanceOf(SpecimenOrObservationBase.class)){
 559                     SpecimenOrObservationBase specimen = CdmBase.deproxy(cdmBase, SpecimenOrObservationBase.class);
 560
 561                     return cdmRepository.getOccurrenceService().saveOrUpdate(specimen);
 562                 }else if (cdmBase.isInstanceOf(Reference.class)){
 563                     return cdmRepository.getReferenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, Reference.class));
 564                 }else if (cdmBase.isInstanceOf(Classification.class)){
 565                     return cdmRepository.getClassificationService().saveOrUpdate(CdmBase.deproxy(cdmBase, Classification.class));
 566                 }else if (cdmBase.isInstanceOf(AgentBase.class)){
 567                     return cdmRepository.getAgentService().saveOrUpdate(CdmBase.deproxy(cdmBase, AgentBase.class));
 568                 }else if (cdmBase.isInstanceOf(Collection.class)){
 569                     return cdmRepository.getCollectionService().saveOrUpdate(CdmBase.deproxy(cdmBase, Collection.class));
 570                 }else if (cdmBase.isInstanceOf(DescriptionBase.class)){
 571                     DescriptionBase description = CdmBase.deproxy(cdmBase, DescriptionBase.class);
 572
 573                     return cdmRepository.getDescriptionService().saveOrUpdate(description);
 574                 }else if (cdmBase.isInstanceOf(TaxonBase.class)){
 575                     return cdmRepository.getTaxonService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonBase.class));
 576                 }else if (cdmBase.isInstanceOf(TaxonName.class)){
 577                     return cdmRepository.getNameService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonName.class));
 578                 }else if (cdmBase.isInstanceOf(TaxonNode.class)){
 579                 return cdmRepository.getTaxonNodeService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonNode.class));
 580             }else{
 581                     throw new IllegalArgumentException("Class not supported in save method: " + CdmBase.deproxy(cdmBase, CdmBase.class).getClass().getSimpleName());
 582                 }
 583
 584             }
 585
 586
 587             protected SpecimenOrObservationBase findExistingSpecimen(String unitId, SpecimenImportStateBase state){
 588                 ICdmRepository cdmAppController = state.getConfig().getCdmAppController();
 589                 if(cdmAppController==null){
 590                     cdmAppController = this;
 591                 }
 592                 FindOccurrencesConfigurator config = new FindOccurrencesConfigurator();
 593                 config.setSignificantIdentifier(unitId);
 594                 List<String> propertyPaths = new ArrayList<>();
 595                 propertyPaths.add("derivedFrom.*");
 596                 config.setPropertyPaths(propertyPaths);
 597                 commitTransaction(state.getTx());
 598                 state.setTx(startTransaction());
 599                 Pager<SpecimenOrObservationBase> existingSpecimens = cdmAppController.getOccurrenceService().findByTitle(config);
 600                 if(!existingSpecimens.getRecords().isEmpty()){
 601                     if(existingSpecimens.getRecords().size()==1){
 602                         return existingSpecimens.getRecords().iterator().next();
 603                     }
 604                 }
 605                 return null;
 606             }
 607
 608             protected abstract void importAssociatedUnits(STATE state, Object item, DerivedUnitFacade derivedUnitFacade);
 609
 610             /**
 611              * getFacade : get the DerivedUnitFacade based on the recordBasis
 612              * @param state
 613              *
 614              * @return DerivedUnitFacade
 615              */
 616             protected DerivedUnitFacade getFacade(STATE state) {
 617                 if (logger.isDebugEnabled()){
 618                     logger.info("getFacade()");
 619                 }
 620                 SpecimenOrObservationType type = null;
 621
 622                 // create specimen
 623                 if (NB((state.getDataHolder().getRecordBasis())) != null) {
 624                     if (state.getDataHolder().getRecordBasis().toLowerCase().startsWith("s") || state.getDataHolder().getRecordBasis().toLowerCase().indexOf("specimen")>-1) {// specimen
 625                         type = SpecimenOrObservationType.PreservedSpecimen;
 626                     }
 627                     if (state.getDataHolder().getRecordBasis().toLowerCase().startsWith("o") ||state.getDataHolder().getRecordBasis().toLowerCase().indexOf("observation")>-1 ) {
 628                         type = SpecimenOrObservationType.Observation;
 629                     }
 630                     if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("fossil")>-1){
 631                         type = SpecimenOrObservationType.Fossil;
 632                     }
 633                     if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("living")>-1) {
 634                         type = SpecimenOrObservationType.LivingSpecimen;
 635                     }
 636                     if (type == null) {
 637                         logger.info("The basis of record does not seem to be known: " + state.getDataHolder().getRecordBasis());
 638                         type = SpecimenOrObservationType.DerivedUnit;
 639                     }
 640                     // TODO fossils?
 641                 } else {
 642                     logger.info("The basis of record is null");
 643                     type = SpecimenOrObservationType.DerivedUnit;
 644                 }
 645                 DerivedUnitFacade derivedUnitFacade = DerivedUnitFacade.NewInstance(type);
 646                 return derivedUnitFacade;
 647             }
 648
 649             /**
 650              * Look if the Institution does already exist
 651              * @param institutionCode: a string with the institutioncode
 652              * @param config : the configurator
 653              * @return the Institution (existing or new)
 654              */
 655             protected Institution getInstitution(String institutionCode, STATE state) {
 656                 SpecimenImportConfiguratorBase config = state.getConfig();
 657                 Institution institution=null;
 658                 institution = (Institution)state.institutions.get(institutionCode);
 659                 if (institution != null){
 660                     return institution;
 661                 }
 662                 List<Institution> institutions;
 663                 try {
 664                     institutions = getAgentService().searchInstitutionByCode(institutionCode);
 665
 666                 } catch (Exception e) {
 667                     institutions = new ArrayList<Institution>();
 668                     logger.warn(e);
 669                 }
 670                 if (institutions.size() > 0 && config.isReuseExistingMetaData()) {
 671                     for (Institution institut:institutions){
 672                         try{
 673                             if (institut.getCode().equalsIgnoreCase(institutionCode)) {
 674                                 institution=institut;
 675                                 break;
 676                             }
 677                         }catch(Exception e){logger.warn("no institution code in the db");}
 678                     }
 679                 }
 680                 if (logger.isDebugEnabled()){
 681                     if(institution !=null) {
 682                         logger.info("getinstitution " + institution.toString());
 683                     }
 684                 }
 685                 if (institution == null){
 686                     // create institution
 687                     institution = Institution.NewInstance();
 688                     institution.setCode(institutionCode);
 689                     institution.setTitleCache(institutionCode, true);
 690                     UUID uuid = save(institution, state);
 691                 }
 692
 693
 694                 state.institutions.put(institutionCode, institution);
 695                 return institution;
 696             }
 697
 698             /**
 699              * Look if the Collection does already exist
 700              * @param collectionCode
 701              * @param collectionCode: a string
 702              * @param config : the configurator
 703              * @return the Collection (existing or new)
 704              */
 705             protected Collection getCollection(Institution institution, String collectionCode, STATE state) {
 706                 SpecimenImportConfiguratorBase config = state.getConfig();
 707                 Collection collection = null;
 708                 List<Collection> collections;
 709                 collection = (Collection) state.collections.get(collectionCode);
 710                 if (collection != null){
 711                     return collection;
 712                 }
 713                 try {
 714                     collections = getCollectionService().searchByCode(collectionCode);
 715                 } catch (Exception e) {
 716                     collections = new ArrayList<Collection>();
 717                 }
 718                 if (collections.size() > 0 && config.isReuseExistingMetaData()) {
 719                     for (Collection coll:collections){
 720                         if (coll.getCode() != null && coll.getInstitute() != null
 721                                 && coll.getCode().equalsIgnoreCase(collectionCode) && coll.getInstitute().equals(institution)) {
 722                             collection = coll;
 723                             break;
 724                         }
 725                     }
 726                 }
 727
 728                 if(collection == null){
 729                     collection =Collection.NewInstance();
 730                     collection.setCode(collectionCode);
 731                     collection.setInstitute(institution);
 732                     collection.setTitleCache(collectionCode);
 733                     UUID uuid = save(collection, state);
 734                 }
 735
 736
 737
 738                 state.collections.put(collectionCode, collection);
 739
 740                 return collection;
 741             }
 742
 743             /**
 744              * @param reference
 745              * @param citationDetail
 746              * @return
 747              */
 748             //FIXME this method is highly critical, because
 749             //  * it will have serious performance and memory problems with large databases
 750             //        (databases may easily have >1 Mio source records)
 751             //  * it does not make sense to search for existing sources and then clone them
 752             //    we need to search for existing references instead and use them (if exist)
 753             //    for our new source.
 754             protected IdentifiableSource getIdentifiableSource(Reference reference, String citationDetail) {
 755
 756               /*  List<IdentifiableSource> issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
 757
 758
 759                 if (reference != null){
 760                     try {
 761                         for (OriginalSourceBase<?> osb: issTmp){
 762                             if (osb.getCitation() != null && osb.getCitation().getTitleCache().equalsIgnoreCase(reference.getTitleCache())){
 763                                 String osbDetail = osb.getCitationMicroReference();
 764                                 if ((StringUtils.isBlank(osbDetail) && StringUtils.isBlank(citationDetail))
 765                                         || (osbDetail != null && osbDetail.equalsIgnoreCase(citationDetail)) ) {
 766 //                                  System.out.println("REFERENCE FOUND RETURN EXISTING SOURCE");
 767                                     return (IdentifiableSource) osb.clone();
 768                                 }
 769                             }
 770                         }
 771                     } catch (CloneNotSupportedException e) {
 772                         throw new RuntimeException(e);
 773                     } catch (Exception e1){
 774                         e1.printStackTrace();
 775                     }
 776                 }
 777         */
 778                 IdentifiableSource sour = IdentifiableSource.NewInstance(OriginalSourceType.Import,null,null, reference,citationDetail);
 779                 return sour;
 780             }
 781
 782             /**
 783              * Add the hierarchy for a Taxon(add higher taxa)
 784              * @param classification
 785              * @param taxon: a taxon to add as a node
 786              * @param state: the ABCD import state
 787              */
 788             protected void addParentTaxon(Taxon taxon, STATE state, boolean preferredFlag, Classification classification){
 789                 INonViralName  nvname = taxon.getName();
 790                 Rank rank = nvname.getRank();
 791                 Taxon genus =null;
 792                 Taxon subgenus =null;
 793                 Taxon species = null;
 794                 Taxon subspecies = null;
 795                 Taxon parent = null;
 796                 if(rank!=null){
 797                     if (rank.isLower(Rank.GENUS() )){
 798                         String genusOrUninomial = nvname.getGenusOrUninomial();
 799                         TaxonName taxonName = getOrCreateTaxonName(genusOrUninomial, Rank.GENUS(), preferredFlag, state, -1);
 800                         genus = getOrCreateTaxonForName(taxonName, state);
 801                         if (genus == null){
 802                             logger.debug("The genus should not be null " + taxonName);
 803                         }
 804                         if (preferredFlag) {
 805                             parent = linkParentChildNode(null, genus, classification, state);
 806                         }
 807
 808                     }
 809                     if (rank.isLower(Rank.SUBGENUS())){
 810                         String prefix = nvname.getGenusOrUninomial();
 811                         String name = nvname.getInfraGenericEpithet();
 812                         if (name != null){
 813                             TaxonName taxonName = getOrCreateTaxonName(prefix+" "+name, Rank.SUBGENUS(), preferredFlag, state, -1);
 814                             subgenus = getOrCreateTaxonForName(taxonName, state);
 815                             if (preferredFlag) {
 816                                 parent = linkParentChildNode(genus, subgenus, classification, state);
 817                             }            }
 818                     }
 819                     if (rank.isLower(Rank.SPECIES())){
 820                         if (subgenus!=null){
 821                             String prefix = nvname.getGenusOrUninomial();
 822                             String name = nvname.getInfraGenericEpithet();
 823                             String spe = nvname.getSpecificEpithet();
 824                             if (spe != null){
 825                                 TaxonName taxonName = getOrCreateTaxonName(prefix+" "+name+" "+spe, Rank.SPECIES(), preferredFlag, state, -1);
 826                                 species = getOrCreateTaxonForName(taxonName, state);
 827                                 if (preferredFlag) {
 828                                     parent = linkParentChildNode(subgenus, species, classification, state);
 829                                 }
 830                             }
 831                         }
 832                         else{
 833                             String prefix = nvname.getGenusOrUninomial();
 834                             String name = nvname.getSpecificEpithet();
 835                             if (name != null){
 836                                 TaxonName taxonName = getOrCreateTaxonName(prefix+" "+name, Rank.SPECIES(), preferredFlag, state, -1);
 837                                 species = getOrCreateTaxonForName(taxonName, state);
 838                                 if (preferredFlag) {
 839                                     parent = linkParentChildNode(genus, species, classification, state);
 840                                 }
 841                             }
 842                         }
 843                     }
 844                     if (rank.isLower(Rank.INFRASPECIES())){
 845                         TaxonName taxonName = getOrCreateTaxonName(nvname.getFullTitleCache(), Rank.SUBSPECIES(), preferredFlag, state, -1);
 846                         subspecies = getOrCreateTaxonForName(taxonName, state);
 847                         if (preferredFlag) {
 848                             parent = linkParentChildNode(species, subspecies, classification, state);
 849                         }
 850                     }
 851                 }
 852                 if (preferredFlag && parent!=taxon ) {
 853                     linkParentChildNode(parent, taxon, classification, state);
 854                 }
 855             }
 856
 857             /**
 858              * Link a parent to a child and save it in the current classification
 859              * @param parent: the higher Taxon
 860              * @param child : the lower (or current) Taxon
 861              * return the Taxon from the new created Node
 862              * @param classification
 863              * @param state
 864              */
 865             protected Taxon linkParentChildNode(Taxon parent, Taxon child, Classification classification, STATE state) {
 866                 TaxonNode node =null;
 867                 if (parent != null) {
 868                     parent = (Taxon) getTaxonService().find(parent.getUuid());
 869                     child = (Taxon) getTaxonService().find(child.getUuid());
 870                     //here we do not have to check if the taxon nodes already exists
 871                     //this is done by classification.addParentChild()
 872                     //do not add child node if it already exists
 873                     if(hasTaxonNodeInClassification(child, classification)){
 874                         return child;
 875                     }
 876                     else{
 877                         node = classification.addParentChild(parent, child, state.getRef(), "");
 878                         save(node, state);
 879                     }
 880                 }
 881                 else {
 882                     if (child == null){
 883                         logger.debug("The child should not be null!");
 884                     }
 885                     child = (Taxon) getTaxonService().find(child.getUuid());
 886                     //do not add child node if it already exists
 887                     if(hasTaxonNodeInClassification(child, classification)){
 888                         return child;
 889                     }
 890                     else{
 891                         node = classification.addChildTaxon(child, state.getRef(), null);
 892                         save(node, state);
 893                     }
 894                 }
 895                 if(node!=null){
 896                     state.getReport().addTaxonNode(node);
 897                     return node.getTaxon();
 898                 }
 899                 String message = "Could not create taxon node for " +child;
 900                 state.getReport().addInfoMessage(message);
 901                 logger.warn(message);
 902                 return null;
 903             }
 904
 905             protected Taxon getOrCreateTaxonForName(TaxonName taxonName, STATE state){
 906                 if (taxonName != null){
 907                 Set<Taxon> acceptedTaxa = taxonName.getTaxa();
 908                 if(acceptedTaxa.size()>0){
 909                     Taxon firstAcceptedTaxon = acceptedTaxa.iterator().next();
 910                     if(acceptedTaxa.size()>1){
 911                         String message = "More than one accepted taxon was found for taxon name: "
 912                                 + taxonName.getTitleCache() + "!\n" + firstAcceptedTaxon + "was chosen for "+state.getDerivedUnitBase();
 913                         state.getReport().addInfoMessage(message);
 914                         logger.warn(message);
 915                     }
 916                     else{
 917                         return firstAcceptedTaxon;
 918                     }
 919                 }
 920                 else{
 921                     Set<TaxonBase> taxonAndSynonyms = taxonName.getTaxonBases();
 922                     for (TaxonBase taxonBase : taxonAndSynonyms) {
 923                         if(taxonBase.isInstanceOf(Synonym.class)){
 924                             Synonym synonym = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
 925                             Taxon acceptedTaxonOfSynonym = synonym.getAcceptedTaxon();
 926                             if(acceptedTaxonOfSynonym == null){
 927                                 String message = "No accepted taxon could be found for taxon name: "
 928                                         + taxonName.getTitleCache()
 929                                         + "!";
 930                                 state.getReport().addInfoMessage(message);
 931                                 logger.warn(message);
 932                             }
 933                             else{
 934                                 return acceptedTaxonOfSynonym;
 935                             }
 936                         }
 937                     }
 938                 }
 939                 Taxon taxon = Taxon.NewInstance(taxonName, state.getRef());
 940                 save(taxon, state);
 941                 state.getReport().addTaxon(taxon);
 942                 logger.info("Created new taxon "+ taxon);
 943                 return taxon;
 944                 }
 945                 return null;
 946
 947             }
 948
 949             private boolean hasTaxonNodeInClassification(Taxon taxon, Classification classification){
 950                 if(taxon.getTaxonNodes()!=null){
 951                     for (TaxonNode node : taxon.getTaxonNodes()){
 952                         if(node.getClassification().equals(classification)){
 953                             return true;
 954                         }
 955                     }
 956                 }
 957                 return false;
 958             }
 959
 960             /**
 961              * HandleIdentifications : get the scientific names present in the ABCD
 962              * document and store link them with the observation/specimen data
 963              * @param state: the current ABCD import state
 964              * @param derivedUnitFacade : the current derivedunitfacade
 965              */
 966             protected void handleIdentifications(STATE state, DerivedUnitFacade derivedUnitFacade) {
 967                SpecimenImportConfiguratorBase config = state.getConfig();
 968
 969
 970                 String scientificName = "";
 971                 boolean preferredFlag = false;
 972
 973                 if (state.getDataHolder().getNomenclatureCode() == ""){
 974                     if (config.getNomenclaturalCode() != null){
 975                         if (config.getNomenclaturalCode() != null){
 976                             state.getDataHolder().setNomenclatureCode(config.getNomenclaturalCode().toString());
 977
 978                         }
 979                     }
 980                 }
 981
 982                 for (int i = 0; i < state.getDataHolder().getIdentificationList().size(); i++) {
 983                     Identification identification = state.getDataHolder().getIdentificationList().get(i);
 984                     scientificName = identification.getScientificName().replaceAll(" et ", " & ");
 985
 986                     String preferred = identification.getPreferred();
 987                     preferredFlag = false;
 988                     if (preferred != null || state.getDataHolder().getIdentificationList().size()==1){
 989                     if (preferred.equals("1") || preferred.toLowerCase().indexOf("true") != -1 || state.getDataHolder().getIdentificationList().size()==1) {
 990                         preferredFlag = true;
 991                     }
 992
 993                     }
 994                     if (identification.getCode() != null){
 995                     if (identification.getCode().indexOf(':') != -1) {
 996                         state.getDataHolder().setNomenclatureCode(identification.getCode().split(COLON)[1]);
 997                     }
 998                     else{
 999                         state.getDataHolder().setNomenclatureCode(identification.getCode());
1000                     }
1001                     }
1002                     TaxonName taxonName = getOrCreateTaxonName(scientificName, null, preferredFlag, state, i);
1003                     Taxon taxon = getOrCreateTaxonForName(taxonName, state);
1004                     addTaxonNode(taxon, state,preferredFlag);
1005                     linkDeterminationEvent(state, taxon, preferredFlag, derivedUnitFacade, identification.getIdentifier(), identification.getDate());
1006                 }
1007             }
1008
1009             /**
1010              * @param taxon : a taxon to add as a node
1011              * @param state : the ABCD import state
1012              */
1013             protected void addTaxonNode(Taxon taxon, STATE state, boolean preferredFlag) {
1014                 SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
1015                 logger.info("link taxon to a taxonNode "+taxon.getTitleCache());
1016                 //only add nodes if not already existing in current classification or default classification
1017
1018                 //check if node exists in current classification
1019                 //NOTE: we cannot use hasTaxonNodeInClassification() here because we are first creating it here
1020                 if (!existsInClassification(taxon,state.getClassification(), state)){
1021                     if(config.isMoveNewTaxaToDefaultClassification()){
1022                         //check if node exists in default classification
1023                         if (!existsInClassification(taxon, state.getDefaultClassification(true), state)){
1024                             addParentTaxon(taxon, state, preferredFlag, state.getDefaultClassification(true));
1025                         }
1026                     }else{
1027                         //add non-existing taxon to current classification
1028                         addParentTaxon(taxon, state, preferredFlag, state.getClassification());
1029                     }
1030
1031                 }
1032             }
1033
1034
1035             private boolean existsInClassification(Taxon taxon, Classification classification, STATE state){
1036                 boolean exist = false;
1037                 ICdmRepository cdmAppController = state.getConfig().getCdmAppController();
1038             if(cdmAppController==null){
1039                 cdmAppController = this;
1040             }
1041             if (classification != null){
1042                 if (!taxon.getTaxonNodes().isEmpty()){
1043                     for (TaxonNode node:taxon.getTaxonNodes()){
1044                         if (node.getClassification().equals(classification)){
1045                             return true;
1046                         }
1047                     }
1048                 }
1049 // we do not need this because we already searched for taxa in db in the previous steps
1050 //              List<UuidAndTitleCache<TaxonNode>> uuidAndTitleCacheOfAllTaxa = cdmAppController.getClassificationService().getTaxonNodeUuidAndTitleCacheOfAcceptedTaxaByClassification(classification.getUuid());
1051 //              if (uuidAndTitleCacheOfAllTaxa != null){
1052 //                      for (UuidAndTitleCache p : uuidAndTitleCacheOfAllTaxa){
1053 //                          try{
1054 //                              if(p.getTitleCache().equals(taxon.getTitleCache())) {
1055 //                                  exist = true;
1056 //                              }
1057 //                          }
1058 //                          catch(Exception e){
1059 //                              logger.warn("TaxonNode doesn't seem to have a taxon");
1060 //                          }
1061 //                      }
1062 //              }
1063             }
1064                 return exist;
1065             }
1066
1067             /**
1068              * join DeterminationEvent to the Taxon Object
1069              * @param state : the ABCD import state
1070              * @param taxon: the current Taxon
1071              * @param preferredFlag :if the current name is preferred
1072              * @param derivedFacade : the derived Unit Facade
1073              */
1074             @SuppressWarnings("rawtypes")
1075         protected void linkDeterminationEvent(STATE state, Taxon taxon, boolean preferredFlag,  DerivedUnitFacade derivedFacade, String identifierStr, String dateStr) {
1076                 SpecimenImportConfiguratorBase config = state.getConfig();
1077                 if (logger.isDebugEnabled()){
1078                     logger.info("start linkdetermination with taxon:" + taxon.getUuid()+", "+taxon);
1079                 }
1080
1081                 DeterminationEvent determinationEvent = DeterminationEvent.NewInstance();
1082                 //determinationEvent.setTaxon(taxon);
1083                 determinationEvent.setTaxonName(taxon.getName());
1084                 determinationEvent.setPreferredFlag(preferredFlag);
1085
1086
1087                 determinationEvent.setIdentifiedUnit(state.getDerivedUnitBase());
1088                 if (state.getPersonStore().get(identifierStr) != null){
1089                     determinationEvent.setActor((AgentBase)state.getPersonStore().get(identifierStr));
1090                 } else if (identifierStr != null){
1091                     Person identifier = Person.NewTitledInstance(identifierStr);
1092                     determinationEvent.setActor(identifier);
1093                 }
1094                 if (dateStr != null){
1095                     determinationEvent.setTimeperiod(TimePeriodParser.parseString(dateStr));
1096                 }
1097                 state.getDerivedUnitBase().addDetermination(determinationEvent);
1098
1099                 if (logger.isDebugEnabled()){
1100                     logger.debug("NB TYPES INFO: "+ state.getDataHolder().getStatusList().size());
1101                 }
1102                 for (SpecimenTypeDesignationStatus specimenTypeDesignationstatus : state.getDataHolder().getStatusList()) {
1103                     if (specimenTypeDesignationstatus != null) {
1104                         if (logger.isDebugEnabled()){
1105                             logger.debug("specimenTypeDesignationstatus :"+ specimenTypeDesignationstatus);
1106                         }
1107
1108                         ICdmRepository cdmAppController = config.getCdmAppController();
1109                         if(cdmAppController == null){
1110                             cdmAppController = this;
1111                         }
1112                         specimenTypeDesignationstatus = (SpecimenTypeDesignationStatus) cdmAppController.getTermService().find(specimenTypeDesignationstatus.getUuid());
1113                         //Designation
1114                         TaxonName name = taxon.getName();
1115                         SpecimenTypeDesignation designation = SpecimenTypeDesignation.NewInstance();
1116
1117                         designation.setTypeStatus(specimenTypeDesignationstatus);
1118                         designation.setTypeSpecimen(state.getDerivedUnitBase());
1119                         name.addTypeDesignation(designation, true);
1120                     }
1121                 }
1122                 save(state.getDerivedUnitBase(), state);
1123
1124                 for (String[] fullReference : state.getDataHolder().getReferenceList()) {
1125
1126
1127                     String strReference=fullReference[0];
1128                     String citationDetail = fullReference[1];
1129                     String citationURL = fullReference[2];
1130                     List<Reference> references = getReferenceService().listByTitleWithRestrictions(Reference.class, "strReference", MatchMode.EXACT, null, null, null, null, null);
1131
1132                     if (!references.isEmpty()){
1133                         Reference reference = null;
1134                         for (Reference refe: references) {
1135                             if (refe.getTitleCache().equalsIgnoreCase(strReference)) {
1136                                 reference =refe;
1137                                 break;
1138                             }
1139                         }
1140                         if (reference ==null){
1141                             reference = ReferenceFactory.newGeneric();
1142                             reference.setTitleCache(strReference, true);
1143                             save(reference, state);
1144                         }
1145                         determinationEvent.addReference(reference);
1146                     }
1147                 }
1148                 save(state.getDerivedUnitBase(), state);
1149
1150                 if (config.isAddIndividualsAssociationsSuchAsSpecimenAndObservations() && preferredFlag) {
1151                     //do not add IndividualsAssociation to non-preferred taxa
1152                     if (logger.isDebugEnabled()){
1153                         logger.debug("isDoCreateIndividualsAssociations");
1154                     }
1155
1156                     makeIndividualsAssociation(state, taxon, determinationEvent);
1157
1158                     save(state.getDerivedUnitBase(), state);
1159                 }
1160             }
1161
1162             /**
1163              * create and link each association (specimen, observation..) to the accepted taxon
1164              * @param state : the ABCD import state
1165              * @param taxon: the current Taxon
1166              * @param determinationEvent:the determinationevent
1167              */
1168             protected void makeIndividualsAssociation(STATE state, Taxon taxon, DeterminationEvent determinationEvent) {
1169                 SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
1170                 SpecimenUserInteraction sui = config.getSpecimenUserInteraction();
1171
1172                 if (logger.isDebugEnabled()){
1173                     logger.info("MAKE INDIVIDUALS ASSOCIATION");
1174                 }
1175
1176                 TaxonDescription taxonDescription = null;
1177                 Set<TaxonDescription> descriptions= taxon.getDescriptions();
1178                if (!descriptions.isEmpty()){ taxonDescription = descriptions.iterator().next();}
1179
1180 //                  for (TaxonDescription description : descriptions){
1181 //                      Set<IdentifiableSource> sources =  new HashSet<>();
1182 //                      sources.addAll(description.getTaxon().getSources());
1183 //                      sources.addAll(description.getSources());
1184 //                      for (IdentifiableSource source:sources){
1185 //                          if(state.getRef().equals(source.getCitation())) {
1186 //                              taxonDescription = description;
1187 //                          }
1188 //                      }
1189 //                  }
1190               //  }
1191                 if (taxonDescription == null){
1192                     taxonDescription = TaxonDescription.NewInstance(taxon, false);
1193                     if(sourceNotLinkedToElement(taxonDescription,state.getRef(),null)) {
1194                         taxonDescription.addSource(OriginalSourceType.Import, null, null, state.getRef(), null);
1195                     }
1196                     state.setDescriptionGroup(taxonDescription);
1197                     taxon.addDescription(taxonDescription);
1198                 }
1199
1200                 //PREPARE REFERENCE QUESTIONS
1201
1202                 Map<String,OriginalSourceBase<?>> sourceMap = new HashMap<String, OriginalSourceBase<?>>();
1203
1204                 List<IdentifiableSource> issTmp = new ArrayList<>();//getCommonService().list(IdentifiableSource.class, null, null, null, null);
1205                 List<DescriptionElementSource> issTmp2 = new ArrayList<>();//getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1206
1207                 Set<OriginalSourceBase> osbSet = new HashSet<OriginalSourceBase>();
1208                 if(issTmp2!=null) {
1209                     osbSet.addAll(issTmp2);
1210                 }
1211                 if(issTmp!=null) {
1212                     osbSet.addAll(issTmp);
1213                 }
1214
1215
1216                 addToSourceMap(sourceMap, osbSet);
1217
1218 //              if (((Abcd206ImportConfigurator) state.getConfig()).isInteractWithUser()){
1219 //                  List<OriginalSourceBase<?>> res = null;
1220 //                  if(!state.isDescriptionSourcesSet()){
1221 //                      res = sui.askForSource(sourceMap, "the description group ("+taxon+")",
1222 //                              "The current reference is "+state.getRef().getTitleCache(),getReferenceService(), state.getDataHolder().docSources);
1223 //                      state.setDescriptionRefs(res);
1224 //                      state.setDescriptionSourcesSet(true);
1225 //                  }
1226 //                  else{
1227 //                      res=state.getDescriptionRefs();
1228 //                  }
1229 //                  if(res !=null) {
1230 //                      for (OriginalSourceBase<?> sour:res){
1231 //                          if(sour.isInstanceOf(IdentifiableSource.class)){
1232 //                              try {
1233 //                                  if(sourceNotLinkedToElement(taxonDescription,sour)) {
1234 //                                      taxonDescription.addSource((IdentifiableSource)sour.clone());
1235 //                                  }
1236 //                              } catch (CloneNotSupportedException e) {
1237 //                                  logger.warn("no cloning?");
1238 //                              }
1239 //                          }else{
1240 //                              if(sourceNotLinkedToElement(taxonDescription,sour)) {
1241 //                                  taxonDescription.addSource(OriginalSourceType.Import,null, null, sour.getCitation(),sour.getCitationMicroReference());
1242 //                              }
1243 //                          }
1244 //                      }
1245 //                  }
1246 //              }
1247 //              else {
1248                     if(sourceNotLinkedToElement(taxonDescription,state.getRef(),null)) {
1249                         taxonDescription.addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1250                     }
1251 //              }
1252                 state.setDescriptionGroup(taxonDescription);
1253
1254                 IndividualsAssociation indAssociation = IndividualsAssociation.NewInstance();
1255                 Feature feature = makeFeature(state.getDerivedUnitBase());
1256                 indAssociation.setAssociatedSpecimenOrObservation(state.getDerivedUnitBase());
1257                 indAssociation.setFeature(feature);
1258 //
1259 //              if (((Abcd206ImportConfigurator) state.getConfig()).isInteractWithUser()){
1260 //                  sourceMap = new HashMap<String, OriginalSourceBase<?>>();
1261 //
1262 //                  issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
1263 //                  issTmp2 = getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1264 //
1265 //                  osbSet = new HashSet<OriginalSourceBase>();
1266 //                  if(issTmp2!=null) {
1267 //                      osbSet.addAll(issTmp2);
1268 //                  }
1269 //                  if(issTmp!=null) {
1270 //                      osbSet.addAll(issTmp);
1271 //                  }
1272 //
1273 //
1274 //                  addToSourceMap(sourceMap, osbSet);
1275 //
1276 //                  List<OriginalSourceBase<?>> sources =null;
1277 //                  if(!state.isAssociationSourcesSet()) {
1278 //                      sources = sui.askForSource(sourceMap,  "descriptive element (association) ",taxon.toString(),
1279 //                              getReferenceService(),state.getDataHolder().getDocSources());
1280 //                      state.setAssociationRefs(sources);
1281 //                      state.setAssociationSourcesSet(true);
1282 //                  }
1283 //                  else{
1284 //                      sources=state.getAssociationRefs();
1285 //                  }
1286 //                  if(sources !=null) {
1287 //                      for (OriginalSourceBase<?> source: sources) {
1288 //                          if(source !=null) {
1289 //                              if(source.isInstanceOf(DescriptionElementSource.class)){
1290 //                                  try {
1291 //                                      if(sourceNotLinkedToElement(indAssociation,source)) {
1292 //                                          indAssociation.addSource((DescriptionElementSource)source.clone());
1293 //                                      }
1294 //                                  } catch (CloneNotSupportedException e) {
1295 //                                      logger.warn("clone forbidden?");
1296 //                                  }
1297 //                              }else{
1298 //                                  if(sourceNotLinkedToElement(indAssociation,source)) {
1299 //                                      indAssociation.addSource(OriginalSourceType.Import,null, null, source.getCitation(),source.getCitationMicroReference());
1300 //                                  }
1301 //                                  try {
1302 //                                      if(sourceNotLinkedToElement(state.getDerivedUnitBase(), source)) {
1303 //                                          state.getDerivedUnitBase().addSource((IdentifiableSource) source.clone());
1304 //                                      }
1305 //                                  } catch (CloneNotSupportedException e) {
1306 //                                      // TODO Auto-generated catch block
1307 //                                      e.printStackTrace();
1308 //                                  }
1309 //                              }
1310 //
1311 //                          }
1312 //                      }
1313 //                  }
1314 //              }else {
1315                     if(sourceNotLinkedToElement(indAssociation,state.getRef(),null)) {
1316                         indAssociation.addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1317                     }
1318                     if(sourceNotLinkedToElement(state.getDerivedUnitBase(), state.getRef(),null)) {
1319                         state.getDerivedUnitBase().addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1320                     }
1321                     for (Reference citation : determinationEvent.getReferences()) {
1322                         if(sourceNotLinkedToElement(indAssociation,citation,null))
1323                         {
1324                             indAssociation.addSource(DescriptionElementSource.NewInstance(OriginalSourceType.Import, null, null, citation, null));
1325                         }
1326                         if(sourceNotLinkedToElement(state.getDerivedUnitBase(), state.getRef(),null)) {
1327                             state.getDerivedUnitBase().addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1328                         }
1329                     }
1330            //     }
1331
1332                 taxonDescription.addElement(indAssociation);
1333
1334                 save(taxonDescription, state);
1335                 save(taxon, state);
1336                 state.getReport().addDerivate(state.getDerivedUnitBase(), config);
1337                 state.getReport().addIndividualAssociation(taxon, state.getDataHolder().getUnitID(), state.getDerivedUnitBase());
1338             }
1339
1340             /**
1341              * @param derivedUnitBase2
1342              * @param ref2
1343              * @param object
1344              * @return
1345              */
1346             private boolean sourceNotLinkedToElement(DerivedUnit derivedUnitBase2, Reference b, String d) {
1347                 Set<IdentifiableSource> linkedSources = derivedUnitBase2.getSources();
1348                 for (IdentifiableSource is:linkedSources){
1349                     Reference a = is.getCitation();
1350                     String c = is.getCitationMicroReference();
1351
1352                     boolean refMatch=false;
1353                     boolean microMatch=false;
1354
1355                     try{
1356                         if (a==null && b==null) {
1357                             refMatch=true;
1358                         }
1359                         if (a!=null && b!=null) {
1360                             if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1361                                 refMatch=true;
1362                             }
1363                         }
1364                     }catch(Exception e){}
1365
1366
1367                     try{
1368                         if (c==null && d==null) {
1369                             microMatch=true;
1370                         }
1371                         if(c!=null && d!=null) {
1372                             if(c.equalsIgnoreCase(d)) {
1373                                 microMatch=true;
1374                             }
1375                         }
1376                     }
1377                     catch(Exception e){}
1378
1379                     if (microMatch && refMatch) {
1380                         return false;
1381                     }
1382
1383
1384                 }
1385                 return true;
1386             }
1387
1388             private <T extends OriginalSourceBase<?>> boolean  sourceNotLinkedToElement(ISourceable<T> sourcable, Reference reference, String microReference) {
1389                 Set<T> linkedSources = sourcable.getSources();
1390                 for (T is:linkedSources){
1391                     Reference unitReference = is.getCitation();
1392                     String unitMicroReference = is.getCitationMicroReference();
1393
1394                     boolean refMatch=false;
1395                     boolean microMatch=false;
1396
1397                     try{
1398                         if (unitReference==null && reference==null) {
1399                             refMatch=true;
1400                         }
1401                         if (unitReference!=null && reference!=null) {
1402                             if (unitReference.getTitleCache().equalsIgnoreCase(reference.getTitleCache())) {
1403                                 refMatch=true;
1404                             }
1405                         }
1406                     }catch(Exception e){}
1407
1408                     try{
1409                         if (unitMicroReference==null && microReference==null) {
1410                             microMatch=true;
1411                         }
1412                         if(unitMicroReference!=null && microReference!=null) {
1413                             if(unitMicroReference.equalsIgnoreCase(microReference)) {
1414                                 microMatch=true;
1415                             }
1416                         }
1417                     }
1418                     catch(Exception e){}
1419
1420                     if (microMatch && refMatch) {
1421                         return false;
1422                     }
1423                 }
1424                 return true;
1425             }
1426
1427             /**
1428              * look for the Feature object (FieldObs, Specimen,...)
1429              * @param unit : a specimen or obersvation base
1430              * @return the corresponding Feature
1431              */
1432             private Feature makeFeature(SpecimenOrObservationBase<?> unit) {
1433                 SpecimenOrObservationType type = unit.getRecordBasis();
1434
1435
1436
1437                 if (type.isFeatureObservation()){
1438                     return Feature.OBSERVATION();
1439                 }else if (type.isFeatureSpecimen()){
1440                     return Feature.SPECIMEN();
1441                 }else if (type == SpecimenOrObservationType.DerivedUnit){
1442                     return Feature.OBSERVATION();
1443                     //            return getFeature("Specimen or observation");
1444                 }else{
1445                     String message = "Unhandled record basis '%s' for defining individuals association feature type. Use default.";
1446                     logger.warn(String.format(message, type.getMessage()));
1447                     return Feature.OBSERVATION();
1448                     //            return getFeature("Specimen or observation");
1449
1450                 }
1451             }
1452
1453
1454             /**
1455              * @param sourceMap
1456              * @param osbSet
1457              */
1458             protected void addToSourceMap(Map<String, OriginalSourceBase<?>> sourceMap, Set<OriginalSourceBase> osbSet) {
1459                 for( OriginalSourceBase<?> osb:osbSet) {
1460                     if(osb.getCitation()!=null && osb.getCitationMicroReference() !=null  && !osb.getCitationMicroReference().isEmpty()) {
1461                         try{
1462                             sourceMap.put(osb.getCitation().getTitleCache()+ "---"+osb.getCitationMicroReference(),osb);
1463                         }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1464                     } else if(osb.getCitation()!=null){
1465                         try{
1466                             sourceMap.put(osb.getCitation().getTitleCache(),osb);
1467                         }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1468                     }
1469                 }
1470             }
1471
1472
1473 }