cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/SpecimenImportBase.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.specimen;
  11
  12
  13 import java.util.ArrayList;
  14 import java.util.HashMap;
  15 import java.util.HashSet;
  16 import java.util.List;
  17 import java.util.Map;
  18 import java.util.Set;
  19 import java.util.UUID;
  20
  21 import org.apache.log4j.Logger;
  22
  23 import eu.etaxonomy.cdm.api.application.ICdmRepository;
  24 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
  25 import eu.etaxonomy.cdm.api.service.config.FindOccurrencesConfigurator;
  26 import eu.etaxonomy.cdm.api.service.pager.Pager;
  27 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  28 import eu.etaxonomy.cdm.io.common.CdmImportBase;
  29 import eu.etaxonomy.cdm.io.common.IImportConfigurator;
  30 import eu.etaxonomy.cdm.io.specimen.abcd206.in.Identification;
  31 import eu.etaxonomy.cdm.io.specimen.abcd206.in.SpecimenImportReport;
  32 import eu.etaxonomy.cdm.model.agent.AgentBase;
  33 import eu.etaxonomy.cdm.model.agent.Institution;
  34 import eu.etaxonomy.cdm.model.agent.Person;
  35 import eu.etaxonomy.cdm.model.agent.Team;
  36 import eu.etaxonomy.cdm.model.common.CdmBase;
  37 import eu.etaxonomy.cdm.model.common.ISourceable;
  38 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  39 import eu.etaxonomy.cdm.model.common.LanguageString;
  40 import eu.etaxonomy.cdm.model.common.OriginalSourceBase;
  41 import eu.etaxonomy.cdm.model.common.OriginalSourceType;
  42 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  43 import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
  44 import eu.etaxonomy.cdm.model.description.Feature;
  45 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  46 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  47 import eu.etaxonomy.cdm.model.name.INonViralName;
  48 import eu.etaxonomy.cdm.model.name.ITaxonNameBase;
  49 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  50 import eu.etaxonomy.cdm.model.name.Rank;
  51 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
  52 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
  53 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
  54 import eu.etaxonomy.cdm.model.name.TaxonNameFactory;
  55 import eu.etaxonomy.cdm.model.occurrence.Collection;
  56 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  57 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
  58 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
  59 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationType;
  60 import eu.etaxonomy.cdm.model.reference.Reference;
  61 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  62 import eu.etaxonomy.cdm.model.taxon.Classification;
  63 import eu.etaxonomy.cdm.model.taxon.Synonym;
  64 import eu.etaxonomy.cdm.model.taxon.Taxon;
  65 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  66 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  67 import eu.etaxonomy.cdm.persistence.query.MatchMode;
  68 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
  69 import eu.etaxonomy.cdm.strategy.parser.ParserProblem;
  70 import eu.etaxonomy.cdm.strategy.parser.TimePeriodParser;
  71
  72
  73 /**
  74  * @author p.kelbert
  75  * @created 20.10.2008
  76  */
  77 public abstract class SpecimenImportBase<CONFIG extends IImportConfigurator, STATE extends SpecimenImportStateBase>
  78         extends CdmImportBase<CONFIG, STATE> {
  79
  80     private static final long serialVersionUID = 4423065367998125678L;
  81     private static final Logger logger = Logger.getLogger(SpecimenImportBase.class);
  82
  83         protected static final UUID SPECIMEN_SCAN_TERM = UUID.fromString("acda15be-c0e2-4ea8-8783-b9b0c4ad7f03");
  84
  85         private static final String COLON = ":";
  86
  87
  88         @Override
  89     protected abstract void doInvoke(STATE state);
  90
  91         /**
  92          * Handle a single unit
  93          * @param state
  94          * @param item
  95          */
  96         protected abstract void handleSingleUnit(STATE state, Object item) ;
  97
  98
  99
 100         protected TaxonNameBase<?, ?> getOrCreateTaxonName(String scientificName, Rank rank, boolean preferredFlag, STATE state, int unitIndexInAbcdFile){
 101             TaxonNameBase<?, ?> taxonName = null;
 102         SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
 103
 104         //check atomised name data for rank
 105         //new name will be created
 106         ITaxonNameBase atomisedTaxonName = null;
 107         if (rank==null && unitIndexInAbcdFile>=0 && ((state.getDataHolder().getAtomisedIdentificationList() != null && !state.getDataHolder().getAtomisedIdentificationList().isEmpty())|| state.getDataHolder().getAtomisedIdentificationList().size() > 0)) {
 108             atomisedTaxonName = setTaxonNameByType(state.getDataHolder().getAtomisedIdentificationList().get(unitIndexInAbcdFile), scientificName, state);
 109             if(atomisedTaxonName!=null){
 110                 rank = atomisedTaxonName.getRank();
 111             }
 112         }
 113         if(config.isReuseExistingTaxaWhenPossible()){
 114             ITaxonNameBase parsedName = atomisedTaxonName;
 115             if(parsedName==null){
 116
 117                 parsedName = parseScientificName(scientificName, state, state.getReport(), rank);
 118
 119             }
 120             atomisedTaxonName = parsedName;
 121             if(config.isIgnoreAuthorship() && parsedName!=null){// && preferredFlag){
 122                 // do not ignore authorship for non-preferred names because they need
 123                 // to be created for the determination history
 124                 String nameCache = TaxonNameBase.castAndDeproxy(parsedName).getNameCache();
 125                 List<TaxonNameBase> names = getNameService().findNamesByNameCache(nameCache, MatchMode.EXACT, null);
 126                 if (!names.isEmpty()){
 127                      taxonName = getBestMatchingName(scientificName, new ArrayList<TaxonNameBase>(names), state);
 128                 }
 129                 if (taxonName == null && !names.isEmpty()){
 130                     taxonName = names.get(0);
 131                 }
 132
 133             } else {
 134                 //search for existing names
 135                 List<TaxonNameBase> names = getNameService().listByTitle(TaxonNameBase.class, scientificName, MatchMode.EXACT, null, null, null, null, null);
 136                 taxonName = getBestMatchingName(scientificName, names, state);
 137                 //still nothing found -> try with the atomised name full title cache
 138                 if(taxonName==null && atomisedTaxonName!=null){
 139                     names = getNameService().listByTitle(TaxonNameBase.class, atomisedTaxonName.getFullTitleCache(), MatchMode.EXACT, null, null, null, null, null);
 140                     taxonName = getBestMatchingName(atomisedTaxonName.getTitleCache(), names, state);
 141                     //still nothing found -> try with the atomised name title cache
 142                     if(taxonName==null){
 143                         names = getNameService().listByTitle(TaxonNameBase.class, atomisedTaxonName.getTitleCache(), MatchMode.EXACT, null, null, null, null, null);
 144                         taxonName = getBestMatchingName(atomisedTaxonName.getTitleCache(), names, state);
 145                     }
 146                 }
 147
 148             }
 149
 150         }
 151
 152         if(taxonName==null && atomisedTaxonName!=null){
 153             taxonName = (TaxonNameBase<?, ?>) atomisedTaxonName;
 154             state.getReport().addName(taxonName);
 155             logger.info("Created new taxon name "+taxonName);
 156             if(taxonName.hasProblem()){
 157                 state.getReport().addInfoMessage(String.format("Created %s with parsing problems", taxonName));
 158             }
 159             if(!atomisedTaxonName.getTitleCache().equals(scientificName)){
 160                 state.getReport().addInfoMessage(String.format("Taxon %s was parsed as %s", scientificName, atomisedTaxonName.getTitleCache()));
 161             }
 162         }
 163         else if(taxonName==null){
 164             //create new taxon name
 165
 166             if (state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNAFP)){
 167                 taxonName = TaxonNameFactory.NewBotanicalInstance(rank);
 168             }else if (state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICZN)){
 169                 taxonName = TaxonNameFactory.NewZoologicalInstance(rank);
 170             }else{
 171                 taxonName = TaxonNameFactory.NewNonViralInstance(rank);
 172             }
 173             taxonName.setFullTitleCache(scientificName,true);
 174             taxonName.setTitleCache(scientificName, true);
 175             state.getReport().addName(taxonName);
 176             logger.info("Created new taxon name "+taxonName);
 177         }
 178         save(taxonName, state);
 179         return taxonName;
 180     }
 181
 182          protected TaxonNameBase<?, ?> getBestMatchingName(String scientificName, java.util.Collection<TaxonNameBase> names, STATE state){
 183                 Set<TaxonNameBase> namesWithAcceptedTaxa = new HashSet<TaxonNameBase>();
 184                 List<TaxonNameBase> namesWithAcceptedTaxaInClassification = new ArrayList<TaxonNameBase>();
 185                 for (TaxonNameBase name : names) {
 186                     if(!name.getTaxa().isEmpty()){
 187                         Set<Taxon> taxa = name.getTaxa();
 188                         for (Taxon taxon:taxa){
 189                             if (!taxon.getTaxonNodes().isEmpty()){
 190                                 //use only taxa included in a classification
 191                                 for (TaxonNode node:taxon.getTaxonNodes()){
 192                                     if (state.getClassification() != null && node.getClassification().equals(state.getClassification())){
 193                                         namesWithAcceptedTaxaInClassification.add(name);
 194                                     }else {
 195                                         namesWithAcceptedTaxa.add(name);
 196                                     }
 197                                 }
 198
 199                             }
 200                         }
 201
 202                     }
 203                 }
 204                 String message = String.format("More than one taxon name was found for %s, maybe in other classifications!", scientificName);
 205                 //check for names with accepted taxa in classification
 206                 if(namesWithAcceptedTaxaInClassification.size()>0){
 207                 if(namesWithAcceptedTaxaInClassification.size()>1){
 208
 209                     state.getReport().addInfoMessage(message);
 210                     logger.warn(message);
 211                     return null;
 212                 }
 213                 return namesWithAcceptedTaxaInClassification.iterator().next();
 214             }
 215               //check for any names with accepted taxa
 216                 if(namesWithAcceptedTaxa.size()>0){
 217                     if(namesWithAcceptedTaxa.size()>1){
 218
 219                         state.getReport().addInfoMessage(message);
 220                         logger.warn(message);
 221                         return null;
 222                     }
 223                     return namesWithAcceptedTaxa.iterator().next();
 224                 }
 225                 //no names with accepted taxa found -> check accepted taxa of synonyms
 226                 List<Taxon> taxaFromSynonyms = new ArrayList<>();
 227                 for (TaxonNameBase name : names) {
 228                     Set<TaxonBase> taxonBases = name.getTaxonBases();
 229                     for (TaxonBase taxonBase : taxonBases) {
 230                         if(taxonBase.isInstanceOf(Synonym.class)){
 231                             Synonym synonym = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
 232                             taxaFromSynonyms.add(synonym.getAcceptedTaxon());
 233                         }
 234                     }
 235                 }
 236                 if(taxaFromSynonyms.size()>0){
 237                     if(taxaFromSynonyms.size()>1){
 238                         state.getReport().addInfoMessage(message);
 239                         logger.warn(message);
 240                         return null;
 241                     }
 242                     return taxaFromSynonyms.iterator().next().getName();
 243                 }
 244                 //no accepted and no synonyms -> return one of the names and create a new taxon
 245                 if (names.isEmpty()){
 246                     return null;
 247                 }else{
 248                     return names.iterator().next();
 249                 }
 250             }
 251          /**
 252              * Parse automatically the scientific name
 253              * @param scientificName the scientific name to parse
 254              * @param state the current import state
 255              * @param report the import report
 256              * @return a parsed name
 257              */
 258
 259             protected ITaxonNameBase parseScientificName(String scientificName, STATE state, SpecimenImportReport report, Rank rank) {
 260
 261                 NonViralNameParserImpl nvnpi = NonViralNameParserImpl.NewInstance();
 262                 ITaxonNameBase taxonName = null;
 263                 boolean problem = false;
 264
 265                 if (logger.isDebugEnabled()){
 266                     logger.debug("parseScientificName " + state.getDataHolder().getNomenclatureCode().toString());
 267                 }
 268
 269                 if (state.getDataHolder().getNomenclatureCode().toString().equals("Zoological") || state.getDataHolder().getNomenclatureCode().toString().contains("ICZN")) {
 270                     taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICZN, rank);
 271                     if (taxonName.hasProblem()) {
 272                         problem = true;
 273                     }
 274                 }
 275                 else if (state.getDataHolder().getNomenclatureCode().toString().equals("Botanical") || state.getDataHolder().getNomenclatureCode().toString().contains("ICBN")) {
 276                     taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNAFP, rank);
 277                     if (taxonName.hasProblem()) {
 278                         problem = true;
 279                     }
 280                 }
 281                 else if (state.getDataHolder().getNomenclatureCode().toString().equals("Bacterial") || state.getDataHolder().getNomenclatureCode().toString().contains("ICBN")) {
 282                     taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNB, rank);
 283                     if (taxonName.hasProblem()) {
 284                         problem = true;
 285                     }
 286                 }
 287                 else if (state.getDataHolder().getNomenclatureCode().toString().equals("Cultivar") || state.getDataHolder().getNomenclatureCode().toString().contains("ICNCP")) {
 288                     taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNCP, rank);
 289                     if (taxonName.hasProblem()) {
 290                         problem = true;
 291                     }
 292                 }
 293                 if (problem) {
 294                     String message = String.format("Parsing problems for %s", scientificName);
 295                     if(taxonName!=null){
 296                         for (ParserProblem parserProblem : taxonName.getParsingProblems()) {
 297                             message += "\n\t- "+parserProblem;
 298                         }
 299                     }
 300                     report.addInfoMessage(message);
 301                     logger.info(message);
 302                 }
 303                 return taxonName;
 304
 305             }
 306
 307             /**
 308              * Create the name without automatic parsing, either because it failed, or because the user deactivated it.
 309              * The name is built upon the ABCD fields
 310              * @param atomisedMap : the ABCD atomised fields
 311              * @param fullName : the full scientific name
 312              * @param state
 313              * @return the corresponding Botanical or Zoological or... name
 314              */
 315             protected TaxonNameBase<?,?> setTaxonNameByType(
 316                     HashMap<String, String> atomisedMap, String fullName, STATE state) {
 317                 boolean problem = false;
 318                 if (logger.isDebugEnabled()){
 319                     logger.debug("settaxonnamebytype " + state.getDataHolder().getNomenclatureCode().toString());
 320                 }
 321
 322                 if (state.getDataHolder().getNomenclatureCode().equals("Zoological") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICZN.getUuid())) {
 323                     TaxonNameBase<?,?> taxonName = TaxonNameFactory.NewZoologicalInstance(null);
 324                     taxonName.setFullTitleCache(fullName, true);
 325                     taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
 326                     taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
 327                     taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap,"SpeciesEpithet")));
 328                     taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap,"SubspeciesEpithet")));
 329
 330                     if (taxonName.getGenusOrUninomial() != null){
 331                         taxonName.setRank(Rank.GENUS());
 332                     }
 333
 334                     if (taxonName.getInfraGenericEpithet() != null){
 335                         taxonName.setRank(Rank.SUBGENUS());
 336                     }
 337
 338                     if (taxonName.getSpecificEpithet() != null){
 339                         taxonName.setRank(Rank.SPECIES());
 340                     }
 341
 342                     if (taxonName.getInfraSpecificEpithet() != null){
 343                         taxonName.setRank(Rank.SUBSPECIES());
 344                     }
 345
 346                     Team team = null;
 347                     if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 348                         team = Team.NewInstance();
 349                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
 350                     }
 351                     else {
 352                         if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 353                             team = Team.NewInstance();
 354                             team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
 355                         }
 356                     }
 357                     if (team != null) {
 358                         taxonName.setBasionymAuthorship(team);
 359                     }
 360                     else {
 361                         if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 362                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
 363                         }
 364                         else if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 365                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamAndYear"));
 366                         }
 367                     }
 368                     if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
 369                         team = Team.NewInstance();
 370                         team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
 371                         taxonName.setCombinationAuthorship(team);
 372                     }
 373                     if (taxonName.hasProblem()) {
 374                         logger.info("pb ICZN");
 375                         problem = true;
 376                     }
 377                     else {
 378                         return taxonName;
 379                     }
 380                 }
 381                 else if (state.getDataHolder().getNomenclatureCode().equals("Botanical") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNAFP.getUuid())) {
 382                     TaxonNameBase taxonName = (TaxonNameBase)parseScientificName(fullName, state, state.getReport(), null);
 383                     if (taxonName != null){
 384                         return taxonName;
 385                     }
 386                     else{
 387                         taxonName = TaxonNameFactory.NewBotanicalInstance(null);
 388                     }
 389                     taxonName.setFullTitleCache(fullName, true);
 390                     taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
 391                     taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "FirstEpithet")));
 392                     taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "InfraSpeEpithet")));
 393                     try {
 394                         taxonName.setRank(Rank.getRankByName(getFromMap(atomisedMap, "Rank")));
 395                     } catch (Exception e) {
 396                         if (taxonName.getInfraSpecificEpithet() != null){
 397                             taxonName.setRank(Rank.SUBSPECIES());
 398                         }
 399                         else if (taxonName.getSpecificEpithet() != null){
 400                             taxonName.setRank(Rank.SPECIES());
 401                         }
 402                         else if (taxonName.getInfraGenericEpithet() != null){
 403                             taxonName.setRank(Rank.SUBGENUS());
 404                         }
 405                         else if (taxonName.getGenusOrUninomial() != null){
 406                             taxonName.setRank(Rank.GENUS());
 407                         }
 408                     }
 409                     Team team = null;
 410                     if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 411                         team = Team.NewInstance();
 412                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
 413                         taxonName.setBasionymAuthorship(team);
 414                     }
 415                     if (getFromMap(atomisedMap, "AuthorTeam") != null) {
 416                         team = Team.NewInstance();
 417                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeam"), true);
 418                         taxonName.setCombinationAuthorship(team);
 419                     }
 420                     if (team == null) {
 421                         if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 422                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
 423                         }
 424                         else if (getFromMap(atomisedMap, "AuthorTeam") != null) {
 425                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeam"));
 426                         }
 427                     }
 428                     if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
 429                         team = Team.NewInstance();
 430                         team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
 431                         taxonName.setCombinationAuthorship(team);
 432                     }
 433                     if (taxonName.hasProblem()) {
 434                         logger.info("pb ICBN");
 435                         problem = true;
 436                     }
 437                     else {
 438                         return taxonName;
 439                     }
 440                 }
 441                 else if (state.getDataHolder().getNomenclatureCode().equals("Bacterial") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNB.getUuid())) {
 442                     TaxonNameBase taxonName = TaxonNameFactory.NewBacterialInstance(null);
 443                     taxonName.setFullTitleCache(fullName, true);
 444                     taxonName.setGenusOrUninomial(getFromMap(atomisedMap, "Genus"));
 445                     taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
 446                     taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "Species")));
 447                     taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "SubspeciesEpithet")));
 448
 449                     if (taxonName.getGenusOrUninomial() != null){
 450                         taxonName.setRank(Rank.GENUS());
 451                     }
 452                     else if (taxonName.getInfraGenericEpithet() != null){
 453                         taxonName.setRank(Rank.SUBGENUS());
 454                     }
 455                     else if (taxonName.getSpecificEpithet() != null){
 456                         taxonName.setRank(Rank.SPECIES());
 457                     }
 458                     else if (taxonName.getInfraSpecificEpithet() != null){
 459                         taxonName.setRank(Rank.SUBSPECIES());
 460                     }
 461
 462                     if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 463                         Team team = Team.NewInstance();
 464                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
 465                         taxonName.setCombinationAuthorship(team);
 466                     }
 467                     if (getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear") != null) {
 468                         Team team = Team.NewInstance();
 469                         team.setTitleCache(getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear"), true);
 470                         taxonName.setBasionymAuthorship(team);
 471                     }
 472                     if (taxonName.hasProblem()) {
 473                         logger.info("pb ICNB");
 474                         problem = true;
 475                     }
 476                     else {
 477                         return taxonName;
 478                     }
 479                 }
 480                 else if (state.getDataHolder().getNomenclatureCode().equals("Cultivar")) {
 481                     TaxonNameBase taxonName = TaxonNameFactory.NewCultivarInstance(null);
 482
 483                     if (taxonName.hasProblem()) {
 484                         logger.info("pb ICNCP");
 485                         problem = true;
 486                     }
 487                     else {
 488                         return taxonName;
 489                     }
 490                     return taxonName;
 491                 }
 492
 493                 if (problem) {
 494                     logger.info("Problem im setTaxonNameByType ");
 495                     TaxonNameBase<?,?> taxonName = TaxonNameFactory.NewNonViralInstance(null);
 496                     taxonName.setFullTitleCache(fullName, true);
 497                     return taxonName;
 498                 }
 499                 TaxonNameBase<?,?> tn = TaxonNameFactory.NewNonViralInstance(null);
 500                 return tn;
 501             }
 502
 503             /**
 504              * Get a formated string from a hashmap
 505              * @param atomisedMap
 506              * @param key
 507              * @return
 508              */
 509             private String getFromMap(HashMap<String, String> atomisedMap, String key) {
 510                 String value = null;
 511                 if (atomisedMap.containsKey(key)) {
 512                     value = atomisedMap.get(key);
 513                 }
 514
 515                 try {
 516                     if (value != null && key.matches(".*Year.*")) {
 517                         value = value.trim();
 518                         if (value.matches("[a-z A-Z ]*[0-9]{4}$")) {
 519                             String tmp = value.split("[0-9]{4}$")[0];
 520                             int year = Integer.parseInt(value.split(tmp)[1]);
 521                             if (year >= 1752) {
 522                                 value = tmp;
 523                             }
 524                             else {
 525                                 value = null;
 526                             }
 527                         }
 528                         else {
 529                             value = null;
 530                         }
 531                     }
 532                 }
 533                 catch (Exception e) {
 534                     value = null;
 535                 }
 536                 return value;
 537             }
 538
 539             /**
 540              * Very fast and dirty implementation to allow handling of transient objects as described in
 541              * https://dev.e-taxonomy.eu/trac/ticket/3726
 542              *
 543              * Not yet complete.
 544              *
 545              * @param cdmBase
 546              * @param state
 547              */
 548             protected UUID save(CdmBase cdmBase, SpecimenImportStateBase state) {
 549                 ICdmRepository cdmRepository = state.getConfig().getCdmAppController();
 550                 if (cdmRepository == null){
 551                     cdmRepository = this;
 552                 }
 553
 554                 if (cdmBase.isInstanceOf(LanguageString.class)){
 555                     return cdmRepository.getTermService().saveLanguageData(CdmBase.deproxy(cdmBase, LanguageString.class));
 556                 }else if (cdmBase.isInstanceOf(SpecimenOrObservationBase.class)){
 557                     return cdmRepository.getOccurrenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, SpecimenOrObservationBase.class));
 558                 }else if (cdmBase.isInstanceOf(Reference.class)){
 559                     return cdmRepository.getReferenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, Reference.class));
 560                 }else if (cdmBase.isInstanceOf(Classification.class)){
 561                     return cdmRepository.getClassificationService().saveOrUpdate(CdmBase.deproxy(cdmBase, Classification.class));
 562                 }else if (cdmBase.isInstanceOf(AgentBase.class)){
 563                     return cdmRepository.getAgentService().saveOrUpdate(CdmBase.deproxy(cdmBase, AgentBase.class));
 564                 }else if (cdmBase.isInstanceOf(Collection.class)){
 565                     return cdmRepository.getCollectionService().saveOrUpdate(CdmBase.deproxy(cdmBase, Collection.class));
 566                 }else if (cdmBase.isInstanceOf(DescriptionBase.class)){
 567                     return cdmRepository.getDescriptionService().saveOrUpdate(CdmBase.deproxy(cdmBase, DescriptionBase.class));
 568                 }else if (cdmBase.isInstanceOf(TaxonBase.class)){
 569                     return cdmRepository.getTaxonService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonBase.class));
 570                 }else if (cdmBase.isInstanceOf(TaxonNameBase.class)){
 571                     return cdmRepository.getNameService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonNameBase.class));
 572                 }else if (cdmBase.isInstanceOf(TaxonNode.class)){
 573                 return cdmRepository.getTaxonNodeService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonNode.class));
 574             }else{
 575                     throw new IllegalArgumentException("Class not supported in save method: " + CdmBase.deproxy(cdmBase, CdmBase.class).getClass().getSimpleName());
 576                 }
 577
 578             }
 579
 580
 581             protected SpecimenOrObservationBase findExistingSpecimen(String unitId, SpecimenImportStateBase state){
 582                 ICdmRepository cdmAppController = state.getConfig().getCdmAppController();
 583                 if(cdmAppController==null){
 584                     cdmAppController = this;
 585                 }
 586                 FindOccurrencesConfigurator config = new FindOccurrencesConfigurator();
 587                 config.setSignificantIdentifier(unitId);
 588                 Pager<SpecimenOrObservationBase> existingSpecimens = cdmAppController.getOccurrenceService().findByTitle(config);
 589                 if(!existingSpecimens.getRecords().isEmpty()){
 590                     if(existingSpecimens.getRecords().size()==1){
 591                         return existingSpecimens.getRecords().iterator().next();
 592                     }
 593                 }
 594                 return null;
 595             }
 596
 597             protected abstract void importAssociatedUnits(STATE state, Object item, DerivedUnitFacade derivedUnitFacade);
 598
 599             /**
 600              * getFacade : get the DerivedUnitFacade based on the recordBasis
 601              * @param state
 602              *
 603              * @return DerivedUnitFacade
 604              */
 605             protected DerivedUnitFacade getFacade(STATE state) {
 606                 if (logger.isDebugEnabled()){
 607                     logger.info("getFacade()");
 608                 }
 609                 SpecimenOrObservationType type = null;
 610
 611                 // create specimen
 612                 if (NB((state.getDataHolder().getRecordBasis())) != null) {
 613                     if (state.getDataHolder().getRecordBasis().toLowerCase().startsWith("s") || state.getDataHolder().getRecordBasis().toLowerCase().indexOf("specimen")>-1) {// specimen
 614                         type = SpecimenOrObservationType.PreservedSpecimen;
 615                     }
 616                     if (state.getDataHolder().getRecordBasis().toLowerCase().startsWith("o") ||state.getDataHolder().getRecordBasis().toLowerCase().indexOf("observation")>-1 ) {
 617                         type = SpecimenOrObservationType.Observation;
 618                     }
 619                     if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("fossil")>-1){
 620                         type = SpecimenOrObservationType.Fossil;
 621                     }
 622                     if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("living")>-1) {
 623                         type = SpecimenOrObservationType.LivingSpecimen;
 624                     }
 625                     if (type == null) {
 626                         logger.info("The basis of record does not seem to be known: " + state.getDataHolder().getRecordBasis());
 627                         type = SpecimenOrObservationType.DerivedUnit;
 628                     }
 629                     // TODO fossils?
 630                 } else {
 631                     logger.info("The basis of record is null");
 632                     type = SpecimenOrObservationType.DerivedUnit;
 633                 }
 634                 DerivedUnitFacade derivedUnitFacade = DerivedUnitFacade.NewInstance(type);
 635                 return derivedUnitFacade;
 636             }
 637
 638             /**
 639              * Look if the Institution does already exist
 640              * @param institutionCode: a string with the institutioncode
 641              * @param config : the configurator
 642              * @return the Institution (existing or new)
 643              */
 644             protected Institution getInstitution(String institutionCode, STATE state) {
 645                 SpecimenImportConfiguratorBase config = state.getConfig();
 646                 Institution institution=null;
 647                 institution = (Institution)state.institutions.get(institutionCode);
 648                 if (institution != null){
 649                     return institution;
 650                 }
 651                 List<Institution> institutions;
 652                 try {
 653                     institutions = getAgentService().searchInstitutionByCode(institutionCode);
 654
 655                 } catch (Exception e) {
 656                     institutions = new ArrayList<Institution>();
 657                     logger.warn(e);
 658                 }
 659                 if (institutions.size() > 0 && config.isReuseExistingMetaData()) {
 660                     for (Institution institut:institutions){
 661                         try{
 662                             if (institut.getCode().equalsIgnoreCase(institutionCode)) {
 663                                 institution=institut;
 664                                 break;
 665                             }
 666                         }catch(Exception e){logger.warn("no institution code in the db");}
 667                     }
 668                 }
 669                 if (logger.isDebugEnabled()){
 670                     if(institution !=null) {
 671                         logger.info("getinstitution " + institution.toString());
 672                     }
 673                 }
 674                 if (institution == null){
 675                     // create institution
 676                     institution = Institution.NewInstance();
 677                     institution.setCode(institutionCode);
 678                     institution.setTitleCache(institutionCode, true);
 679                     UUID uuid = save(institution, state);
 680                 }
 681
 682
 683                 state.institutions.put(institutionCode, institution);
 684                 return institution;
 685             }
 686
 687             /**
 688              * Look if the Collection does already exist
 689              * @param collectionCode
 690              * @param collectionCode: a string
 691              * @param config : the configurator
 692              * @return the Collection (existing or new)
 693              */
 694             protected Collection getCollection(Institution institution, String collectionCode, STATE state) {
 695                 SpecimenImportConfiguratorBase config = state.getConfig();
 696                 Collection collection = null;
 697                 List<Collection> collections;
 698                 collection = (Collection) state.collections.get(collectionCode);
 699                 if (collection != null){
 700                     return collection;
 701                 }
 702                 try {
 703                     collections = getCollectionService().searchByCode(collectionCode);
 704                 } catch (Exception e) {
 705                     collections = new ArrayList<Collection>();
 706                 }
 707                 if (collections.size() > 0 && config.isReuseExistingMetaData()) {
 708                     for (Collection coll:collections){
 709                         if (coll.getCode() != null && coll.getInstitute() != null
 710                                 && coll.getCode().equalsIgnoreCase(collectionCode) && coll.getInstitute().equals(institution)) {
 711                             collection = coll;
 712                             break;
 713                         }
 714                     }
 715                 }
 716
 717                 if(collection == null){
 718                     collection =Collection.NewInstance();
 719                     collection.setCode(collectionCode);
 720                     collection.setInstitute(institution);
 721                     collection.setTitleCache(collectionCode);
 722                     UUID uuid = save(collection, state);
 723                 }
 724
 725
 726
 727                 state.collections.put(collectionCode, collection);
 728
 729                 return collection;
 730             }
 731
 732             /**
 733              * @param reference
 734              * @param citationDetail
 735              * @return
 736              */
 737             //FIXME this method is highly critical, because
 738             //  * it will have serious performance and memory problems with large databases
 739             //        (databases may easily have >1 Mio source records)
 740             //  * it does not make sense to search for existing sources and then clone them
 741             //    we need to search for existing references instead and use them (if exist)
 742             //    for our new source.
 743             protected IdentifiableSource getIdentifiableSource(Reference reference, String citationDetail) {
 744
 745               /*  List<IdentifiableSource> issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
 746
 747
 748                 if (reference != null){
 749                     try {
 750                         for (OriginalSourceBase<?> osb: issTmp){
 751                             if (osb.getCitation() != null && osb.getCitation().getTitleCache().equalsIgnoreCase(reference.getTitleCache())){
 752                                 String osbDetail = osb.getCitationMicroReference();
 753                                 if ((StringUtils.isBlank(osbDetail) && StringUtils.isBlank(citationDetail))
 754                                         || (osbDetail != null && osbDetail.equalsIgnoreCase(citationDetail)) ) {
 755 //                                  System.out.println("REFERENCE FOUND RETURN EXISTING SOURCE");
 756                                     return (IdentifiableSource) osb.clone();
 757                                 }
 758                             }
 759                         }
 760                     } catch (CloneNotSupportedException e) {
 761                         throw new RuntimeException(e);
 762                     } catch (Exception e1){
 763                         e1.printStackTrace();
 764                     }
 765                 }
 766         */
 767                 IdentifiableSource sour = IdentifiableSource.NewInstance(OriginalSourceType.Import,null,null, reference,citationDetail);
 768                 return sour;
 769             }
 770
 771             /**
 772              * Add the hierarchy for a Taxon(add higher taxa)
 773              * @param classification
 774              * @param taxon: a taxon to add as a node
 775              * @param state: the ABCD import state
 776              */
 777             protected void addParentTaxon(Taxon taxon, STATE state, boolean preferredFlag, Classification classification){
 778                 INonViralName  nvname = taxon.getName();
 779                 Rank rank = nvname.getRank();
 780                 Taxon genus =null;
 781                 Taxon subgenus =null;
 782                 Taxon species = null;
 783                 Taxon subspecies = null;
 784                 Taxon parent = null;
 785                 if(rank!=null){
 786                     if (rank.isLower(Rank.GENUS() )){
 787                         String genusOrUninomial = nvname.getGenusOrUninomial();
 788                         TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(genusOrUninomial, Rank.GENUS(), preferredFlag, state, -1);
 789                         genus = getOrCreateTaxonForName(taxonName, state);
 790                         if (genus == null){
 791                             logger.debug("The genus should not be null " + taxonName);
 792                         }
 793                         if (preferredFlag) {
 794                             parent = linkParentChildNode(null, genus, classification, state);
 795                         }
 796
 797                     }
 798                     if (rank.isLower(Rank.SUBGENUS())){
 799                         String prefix = nvname.getGenusOrUninomial();
 800                         String name = nvname.getInfraGenericEpithet();
 801                         if (name != null){
 802                             TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(prefix+" "+name, Rank.SUBGENUS(), preferredFlag, state, -1);
 803                             subgenus = getOrCreateTaxonForName(taxonName, state);
 804                             if (preferredFlag) {
 805                                 parent = linkParentChildNode(genus, subgenus, classification, state);
 806                             }            }
 807                     }
 808                     if (rank.isLower(Rank.SPECIES())){
 809                         if (subgenus!=null){
 810                             String prefix = nvname.getGenusOrUninomial();
 811                             String name = nvname.getInfraGenericEpithet();
 812                             String spe = nvname.getSpecificEpithet();
 813                             if (spe != null){
 814                                 TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(prefix+" "+name+" "+spe, Rank.SPECIES(), preferredFlag, state, -1);
 815                                 species = getOrCreateTaxonForName(taxonName, state);
 816                                 if (preferredFlag) {
 817                                     parent = linkParentChildNode(subgenus, species, classification, state);
 818                                 }
 819                             }
 820                         }
 821                         else{
 822                             String prefix = nvname.getGenusOrUninomial();
 823                             String name = nvname.getSpecificEpithet();
 824                             if (name != null){
 825                                 TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(prefix+" "+name, Rank.SPECIES(), preferredFlag, state, -1);
 826                                 species = getOrCreateTaxonForName(taxonName, state);
 827                                 if (preferredFlag) {
 828                                     parent = linkParentChildNode(genus, species, classification, state);
 829                                 }
 830                             }
 831                         }
 832                     }
 833                     if (rank.isLower(Rank.INFRASPECIES())){
 834                         TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(nvname.getFullTitleCache(), Rank.SUBSPECIES(), preferredFlag, state, -1);
 835                         subspecies = getOrCreateTaxonForName(taxonName, state);
 836                         if (preferredFlag) {
 837                             parent = linkParentChildNode(species, subspecies, classification, state);
 838                         }
 839                     }
 840                 }
 841                 if (preferredFlag && parent!=taxon ) {
 842                     linkParentChildNode(parent, taxon, classification, state);
 843                 }
 844             }
 845
 846             /**
 847              * Link a parent to a child and save it in the current classification
 848              * @param parent: the higher Taxon
 849              * @param child : the lower (or current) Taxon
 850              * return the Taxon from the new created Node
 851              * @param classification
 852              * @param state
 853              */
 854             protected Taxon linkParentChildNode(Taxon parent, Taxon child, Classification classification, STATE state) {
 855                 TaxonNode node =null;
 856                 if (parent != null) {
 857                     parent = (Taxon) getTaxonService().find(parent.getUuid());
 858                     child = (Taxon) getTaxonService().find(child.getUuid());
 859                     //here we do not have to check if the taxon nodes already exists
 860                     //this is done by classification.addParentChild()
 861                     //do not add child node if it already exists
 862                     if(hasTaxonNodeInClassification(child, classification)){
 863                         return child;
 864                     }
 865                     else{
 866                         node = classification.addParentChild(parent, child, state.getRef(), "");
 867                         save(node, state);
 868                     }
 869                 }
 870                 else {
 871                     if (child == null){
 872                         logger.debug("The child should not be null!");
 873                     }
 874                     child = (Taxon) getTaxonService().find(child.getUuid());
 875                     //do not add child node if it already exists
 876                     if(hasTaxonNodeInClassification(child, classification)){
 877                         return child;
 878                     }
 879                     else{
 880                         node = classification.addChildTaxon(child, state.getRef(), null);
 881                         save(node, state);
 882                     }
 883                 }
 884                 if(node!=null){
 885                     state.getReport().addTaxonNode(node);
 886                     return node.getTaxon();
 887                 }
 888                 String message = "Could not create taxon node for " +child;
 889                 state.getReport().addInfoMessage(message);
 890                 logger.warn(message);
 891                 return null;
 892             }
 893
 894             protected Taxon getOrCreateTaxonForName(TaxonNameBase<?, ?> taxonNameBase, STATE state){
 895                 if (taxonNameBase != null){
 896                 Set<Taxon> acceptedTaxa = taxonNameBase.getTaxa();
 897                 if(acceptedTaxa.size()>0){
 898                     Taxon firstAcceptedTaxon = acceptedTaxa.iterator().next();
 899                     if(acceptedTaxa.size()>1){
 900                         String message = "More than one accepted taxon was found for taxon name: "
 901                                 + taxonNameBase.getTitleCache() + "!\n" + firstAcceptedTaxon + "was chosen for "+state.getDerivedUnitBase();
 902                         state.getReport().addInfoMessage(message);
 903                         logger.warn(message);
 904                     }
 905                     else{
 906                         return firstAcceptedTaxon;
 907                     }
 908                 }
 909                 else{
 910                     Set<TaxonBase> taxonAndSynonyms = taxonNameBase.getTaxonBases();
 911                     for (TaxonBase taxonBase : taxonAndSynonyms) {
 912                         if(taxonBase.isInstanceOf(Synonym.class)){
 913                             Synonym synonym = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
 914                             Taxon acceptedTaxonOfSynonym = synonym.getAcceptedTaxon();
 915                             if(acceptedTaxonOfSynonym == null){
 916                                 String message = "No accepted taxon could be found for taxon name: "
 917                                         + taxonNameBase.getTitleCache()
 918                                         + "!";
 919                                 state.getReport().addInfoMessage(message);
 920                                 logger.warn(message);
 921                             }
 922                             else{
 923                                 return acceptedTaxonOfSynonym;
 924                             }
 925                         }
 926                     }
 927                 }
 928                 Taxon taxon = Taxon.NewInstance(taxonNameBase, state.getRef());
 929                 save(taxon, state);
 930                 state.getReport().addTaxon(taxon);
 931                 logger.info("Created new taxon "+ taxon);
 932                 return taxon;
 933                 }
 934                 return null;
 935
 936             }
 937
 938             private boolean hasTaxonNodeInClassification(Taxon taxon, Classification classification){
 939                 if(taxon.getTaxonNodes()!=null){
 940                     for (TaxonNode node : taxon.getTaxonNodes()){
 941                         if(node.getClassification().equals(classification)){
 942                             return true;
 943                         }
 944                     }
 945                 }
 946                 return false;
 947             }
 948
 949             /**
 950              * HandleIdentifications : get the scientific names present in the ABCD
 951              * document and store link them with the observation/specimen data
 952              * @param state: the current ABCD import state
 953              * @param derivedUnitFacade : the current derivedunitfacade
 954              */
 955             protected void handleIdentifications(STATE state, DerivedUnitFacade derivedUnitFacade) {
 956                SpecimenImportConfiguratorBase config = state.getConfig();
 957
 958
 959                 String scientificName = "";
 960                 boolean preferredFlag = false;
 961
 962                 if (state.getDataHolder().getNomenclatureCode() == ""){
 963                     if (config.getNomenclaturalCode() != null){
 964                         if (config.getNomenclaturalCode() != null){
 965                             state.getDataHolder().setNomenclatureCode(config.getNomenclaturalCode().toString());
 966
 967                         }
 968                     }
 969                 }
 970
 971                 for (int i = 0; i < state.getDataHolder().getIdentificationList().size(); i++) {
 972                     Identification identification = state.getDataHolder().getIdentificationList().get(i);
 973                     scientificName = identification.getScientificName().replaceAll(" et ", " & ");
 974
 975                     String preferred = identification.getPreferred();
 976                     preferredFlag = false;
 977                     if (preferred != null){
 978                     if (preferred.equals("1") || preferred.toLowerCase().indexOf("true") != -1 || state.getDataHolder().getIdentificationList().size()==1) {
 979                         preferredFlag = true;
 980                     }
 981
 982                     }
 983                     if (identification.getCode() != null){
 984                     if (identification.getCode().indexOf(':') != -1) {
 985                         state.getDataHolder().setNomenclatureCode(identification.getCode().split(COLON)[1]);
 986                     }
 987                     else{
 988                         state.getDataHolder().setNomenclatureCode(identification.getCode());
 989                     }
 990                     }
 991                     TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(scientificName, null, preferredFlag, state, i);
 992                     Taxon taxon = getOrCreateTaxonForName(taxonName, state);
 993                     addTaxonNode(taxon, state,preferredFlag);
 994                     linkDeterminationEvent(state, taxon, preferredFlag, derivedUnitFacade, identification.getIdentifier(), identification.getDate());
 995                 }
 996             }
 997
 998             /**
 999              * @param taxon : a taxon to add as a node
1000              * @param state : the ABCD import state
1001              */
1002             protected void addTaxonNode(Taxon taxon, STATE state, boolean preferredFlag) {
1003                 SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
1004                 logger.info("link taxon to a taxonNode "+taxon.getTitleCache());
1005                 //only add nodes if not already existing in current classification or default classification
1006
1007                 //check if node exists in current classification
1008                 //NOTE: we cannot use hasTaxonNodeInClassification() here because we are first creating it here
1009                 if (!existsInClassification(taxon,state.getClassification(), state)){
1010                     if(config.isMoveNewTaxaToDefaultClassification()){
1011                         //check if node exists in default classification
1012                         if (!existsInClassification(taxon, state.getDefaultClassification(), state)){
1013                             addParentTaxon(taxon, state, preferredFlag, state.getDefaultClassification());
1014                         }
1015                     }
1016                     else {
1017                         //add non-existing taxon to current classification
1018                         addParentTaxon(taxon, state, preferredFlag, state.getClassification());
1019                     }
1020                 }
1021             }
1022
1023
1024             private boolean existsInClassification(Taxon taxon, Classification classification, STATE state){
1025                 boolean exist = false;
1026                 ICdmRepository cdmAppController = state.getConfig().getCdmAppController();
1027             if(cdmAppController==null){
1028                 cdmAppController = this;
1029             }
1030             if (classification != null){
1031                 if (!taxon.getTaxonNodes().isEmpty()){
1032                     for (TaxonNode node:taxon.getTaxonNodes()){
1033                         if (node.getClassification().equals(classification)){
1034                             return true;
1035                         }
1036                     }
1037                 }
1038 // we do not need this because we already searched for taxa in db in the previous steps
1039 //              List<UuidAndTitleCache<TaxonNode>> uuidAndTitleCacheOfAllTaxa = cdmAppController.getClassificationService().getTaxonNodeUuidAndTitleCacheOfAcceptedTaxaByClassification(classification.getUuid());
1040 //              if (uuidAndTitleCacheOfAllTaxa != null){
1041 //                      for (UuidAndTitleCache p : uuidAndTitleCacheOfAllTaxa){
1042 //                          try{
1043 //                              if(p.getTitleCache().equals(taxon.getTitleCache())) {
1044 //                                  exist = true;
1045 //                              }
1046 //                          }
1047 //                          catch(Exception e){
1048 //                              logger.warn("TaxonNode doesn't seem to have a taxon");
1049 //                          }
1050 //                      }
1051 //              }
1052             }
1053                 return exist;
1054             }
1055
1056             /**
1057              * join DeterminationEvent to the Taxon Object
1058              * @param state : the ABCD import state
1059              * @param taxon: the current Taxon
1060              * @param preferredFlag :if the current name is preferred
1061              * @param derivedFacade : the derived Unit Facade
1062              */
1063             @SuppressWarnings("rawtypes")
1064         protected void linkDeterminationEvent(STATE state, Taxon taxon, boolean preferredFlag,  DerivedUnitFacade derivedFacade, String identifierStr, String dateStr) {
1065                 SpecimenImportConfiguratorBase config = state.getConfig();
1066                 if (logger.isDebugEnabled()){
1067                     logger.info("start linkdetermination with taxon:" + taxon.getUuid()+", "+taxon);
1068                 }
1069
1070                 DeterminationEvent determinationEvent = DeterminationEvent.NewInstance();
1071                 determinationEvent.setTaxonName(taxon.getName());
1072                 determinationEvent.setPreferredFlag(preferredFlag);
1073
1074
1075                 determinationEvent.setIdentifiedUnit(state.getDerivedUnitBase());
1076                 if (state.getPersonStore().get(identifierStr) != null){
1077                     determinationEvent.setActor((AgentBase)state.getPersonStore().get(identifierStr));
1078                 } else if (identifierStr != null){
1079                     Person identifier = Person.NewTitledInstance(identifierStr);
1080                     determinationEvent.setActor(identifier);
1081                 }
1082                 if (dateStr != null){
1083                     determinationEvent.setTimeperiod(TimePeriodParser.parseString(dateStr));
1084                 }
1085                 state.getDerivedUnitBase().addDetermination(determinationEvent);
1086
1087                 if (logger.isDebugEnabled()){
1088                     logger.debug("NB TYPES INFO: "+ state.getDataHolder().getStatusList().size());
1089                 }
1090                 for (SpecimenTypeDesignationStatus specimenTypeDesignationstatus : state.getDataHolder().getStatusList()) {
1091                     if (specimenTypeDesignationstatus != null) {
1092                         if (logger.isDebugEnabled()){
1093                             logger.debug("specimenTypeDesignationstatus :"+ specimenTypeDesignationstatus);
1094                         }
1095
1096                         ICdmRepository cdmAppController = config.getCdmAppController();
1097                         if(cdmAppController == null){
1098                             cdmAppController = this;
1099                         }
1100                         specimenTypeDesignationstatus = (SpecimenTypeDesignationStatus) cdmAppController.getTermService().find(specimenTypeDesignationstatus.getUuid());
1101                         //Designation
1102                         TaxonNameBase<?,?> name = taxon.getName();
1103                         SpecimenTypeDesignation designation = SpecimenTypeDesignation.NewInstance();
1104
1105                         designation.setTypeStatus(specimenTypeDesignationstatus);
1106                         designation.setTypeSpecimen(state.getDerivedUnitBase());
1107                         name.addTypeDesignation(designation, true);
1108                     }
1109                 }
1110
1111                 for (String[] fullReference : state.getDataHolder().getReferenceList()) {
1112
1113
1114                     String strReference=fullReference[0];
1115                     String citationDetail = fullReference[1];
1116                     String citationURL = fullReference[2];
1117                     List<Reference> references = getReferenceService().listByTitle(Reference.class, "strReference", MatchMode.EXACT, null, null, null, null, null);
1118
1119                     if (!references.isEmpty()){
1120                         Reference reference = null;
1121                         for (Reference refe: references) {
1122                             if (refe.getTitleCache().equalsIgnoreCase(strReference)) {
1123                                 reference =refe;
1124                                 break;
1125                             }
1126                         }
1127                         if (reference ==null){
1128                             reference = ReferenceFactory.newGeneric();
1129                             reference.setTitleCache(strReference, true);
1130                             save(reference, state);
1131                         }
1132                         determinationEvent.addReference(reference);
1133                     }
1134                 }
1135                 save(state.getDerivedUnitBase(), state);
1136
1137                 if (config.isAddIndividualsAssociationsSuchAsSpecimenAndObservations() && preferredFlag) {
1138                     //do not add IndividualsAssociation to non-preferred taxa
1139                     if (logger.isDebugEnabled()){
1140                         logger.debug("isDoCreateIndividualsAssociations");
1141                     }
1142
1143                     makeIndividualsAssociation(state, taxon, determinationEvent);
1144
1145                     save(state.getDerivedUnitBase(), state);
1146                 }
1147             }
1148
1149             /**
1150              * create and link each association (specimen, observation..) to the accepted taxon
1151              * @param state : the ABCD import state
1152              * @param taxon: the current Taxon
1153              * @param determinationEvent:the determinationevent
1154              */
1155             protected void makeIndividualsAssociation(STATE state, Taxon taxon, DeterminationEvent determinationEvent) {
1156                 SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
1157                 SpecimenUserInteraction sui = config.getSpecimenUserInteraction();
1158
1159                 if (logger.isDebugEnabled()){
1160                     logger.info("MAKE INDIVIDUALS ASSOCIATION");
1161                 }
1162
1163                 TaxonDescription taxonDescription = null;
1164                 Set<TaxonDescription> descriptions= taxon.getDescriptions();
1165                if (!descriptions.isEmpty()){ taxonDescription = descriptions.iterator().next();}
1166
1167 //                  for (TaxonDescription description : descriptions){
1168 //                      Set<IdentifiableSource> sources =  new HashSet<>();
1169 //                      sources.addAll(description.getTaxon().getSources());
1170 //                      sources.addAll(description.getSources());
1171 //                      for (IdentifiableSource source:sources){
1172 //                          if(state.getRef().equals(source.getCitation())) {
1173 //                              taxonDescription = description;
1174 //                          }
1175 //                      }
1176 //                  }
1177               //  }
1178                 if (taxonDescription == null){
1179                     taxonDescription = TaxonDescription.NewInstance(taxon, false);
1180                     if(sourceNotLinkedToElement(taxonDescription,state.getRef(),null)) {
1181                         taxonDescription.addSource(OriginalSourceType.Import, null, null, state.getRef(), null);
1182                     }
1183                     state.setDescriptionGroup(taxonDescription);
1184                     taxon.addDescription(taxonDescription);
1185                 }
1186
1187                 //PREPARE REFERENCE QUESTIONS
1188
1189                 Map<String,OriginalSourceBase<?>> sourceMap = new HashMap<String, OriginalSourceBase<?>>();
1190
1191                 List<IdentifiableSource> issTmp = new ArrayList<>();//getCommonService().list(IdentifiableSource.class, null, null, null, null);
1192                 List<DescriptionElementSource> issTmp2 = new ArrayList<>();//getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1193
1194                 Set<OriginalSourceBase> osbSet = new HashSet<OriginalSourceBase>();
1195                 if(issTmp2!=null) {
1196                     osbSet.addAll(issTmp2);
1197                 }
1198                 if(issTmp!=null) {
1199                     osbSet.addAll(issTmp);
1200                 }
1201
1202
1203                 addToSourceMap(sourceMap, osbSet);
1204
1205 //              if (((Abcd206ImportConfigurator) state.getConfig()).isInteractWithUser()){
1206 //                  List<OriginalSourceBase<?>> res = null;
1207 //                  if(!state.isDescriptionSourcesSet()){
1208 //                      res = sui.askForSource(sourceMap, "the description group ("+taxon+")",
1209 //                              "The current reference is "+state.getRef().getTitleCache(),getReferenceService(), state.getDataHolder().docSources);
1210 //                      state.setDescriptionRefs(res);
1211 //                      state.setDescriptionSourcesSet(true);
1212 //                  }
1213 //                  else{
1214 //                      res=state.getDescriptionRefs();
1215 //                  }
1216 //                  if(res !=null) {
1217 //                      for (OriginalSourceBase<?> sour:res){
1218 //                          if(sour.isInstanceOf(IdentifiableSource.class)){
1219 //                              try {
1220 //                                  if(sourceNotLinkedToElement(taxonDescription,sour)) {
1221 //                                      taxonDescription.addSource((IdentifiableSource)sour.clone());
1222 //                                  }
1223 //                              } catch (CloneNotSupportedException e) {
1224 //                                  logger.warn("no cloning?");
1225 //                              }
1226 //                          }else{
1227 //                              if(sourceNotLinkedToElement(taxonDescription,sour)) {
1228 //                                  taxonDescription.addSource(OriginalSourceType.Import,null, null, sour.getCitation(),sour.getCitationMicroReference());
1229 //                              }
1230 //                          }
1231 //                      }
1232 //                  }
1233 //              }
1234 //              else {
1235                     if(sourceNotLinkedToElement(taxonDescription,state.getRef(),null)) {
1236                         taxonDescription.addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1237                     }
1238 //              }
1239                 state.setDescriptionGroup(taxonDescription);
1240
1241                 IndividualsAssociation indAssociation = IndividualsAssociation.NewInstance();
1242                 Feature feature = makeFeature(state.getDerivedUnitBase());
1243                 indAssociation.setAssociatedSpecimenOrObservation(state.getDerivedUnitBase());
1244                 indAssociation.setFeature(feature);
1245 //
1246 //              if (((Abcd206ImportConfigurator) state.getConfig()).isInteractWithUser()){
1247 //                  sourceMap = new HashMap<String, OriginalSourceBase<?>>();
1248 //
1249 //                  issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
1250 //                  issTmp2 = getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1251 //
1252 //                  osbSet = new HashSet<OriginalSourceBase>();
1253 //                  if(issTmp2!=null) {
1254 //                      osbSet.addAll(issTmp2);
1255 //                  }
1256 //                  if(issTmp!=null) {
1257 //                      osbSet.addAll(issTmp);
1258 //                  }
1259 //
1260 //
1261 //                  addToSourceMap(sourceMap, osbSet);
1262 //
1263 //                  List<OriginalSourceBase<?>> sources =null;
1264 //                  if(!state.isAssociationSourcesSet()) {
1265 //                      sources = sui.askForSource(sourceMap,  "descriptive element (association) ",taxon.toString(),
1266 //                              getReferenceService(),state.getDataHolder().getDocSources());
1267 //                      state.setAssociationRefs(sources);
1268 //                      state.setAssociationSourcesSet(true);
1269 //                  }
1270 //                  else{
1271 //                      sources=state.getAssociationRefs();
1272 //                  }
1273 //                  if(sources !=null) {
1274 //                      for (OriginalSourceBase<?> source: sources) {
1275 //                          if(source !=null) {
1276 //                              if(source.isInstanceOf(DescriptionElementSource.class)){
1277 //                                  try {
1278 //                                      if(sourceNotLinkedToElement(indAssociation,source)) {
1279 //                                          indAssociation.addSource((DescriptionElementSource)source.clone());
1280 //                                      }
1281 //                                  } catch (CloneNotSupportedException e) {
1282 //                                      logger.warn("clone forbidden?");
1283 //                                  }
1284 //                              }else{
1285 //                                  if(sourceNotLinkedToElement(indAssociation,source)) {
1286 //                                      indAssociation.addSource(OriginalSourceType.Import,null, null, source.getCitation(),source.getCitationMicroReference());
1287 //                                  }
1288 //                                  try {
1289 //                                      if(sourceNotLinkedToElement(state.getDerivedUnitBase(), source)) {
1290 //                                          state.getDerivedUnitBase().addSource((IdentifiableSource) source.clone());
1291 //                                      }
1292 //                                  } catch (CloneNotSupportedException e) {
1293 //                                      // TODO Auto-generated catch block
1294 //                                      e.printStackTrace();
1295 //                                  }
1296 //                              }
1297 //
1298 //                          }
1299 //                      }
1300 //                  }
1301 //              }else {
1302                     if(sourceNotLinkedToElement(indAssociation,state.getRef(),null)) {
1303                         indAssociation.addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1304                     }
1305                     if(sourceNotLinkedToElement(state.getDerivedUnitBase(), state.getRef(),null)) {
1306                         state.getDerivedUnitBase().addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1307                     }
1308                     for (Reference citation : determinationEvent.getReferences()) {
1309                         if(sourceNotLinkedToElement(indAssociation,citation,null))
1310                         {
1311                             indAssociation.addSource(DescriptionElementSource.NewInstance(OriginalSourceType.Import, null, null, citation, null));
1312                         }
1313                         if(sourceNotLinkedToElement(state.getDerivedUnitBase(), state.getRef(),null)) {
1314                             state.getDerivedUnitBase().addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1315                         }
1316                     }
1317            //     }
1318
1319                 taxonDescription.addElement(indAssociation);
1320
1321                 save(taxonDescription, state);
1322                 save(taxon, state);
1323                 state.getReport().addDerivate(state.getDerivedUnitBase(), config);
1324                 state.getReport().addIndividualAssociation(taxon, state.getDataHolder().getUnitID(), state.getDerivedUnitBase());
1325             }
1326
1327             /**
1328              * @param derivedUnitBase2
1329              * @param ref2
1330              * @param object
1331              * @return
1332              */
1333             private boolean sourceNotLinkedToElement(DerivedUnit derivedUnitBase2, Reference b, String d) {
1334                 Set<IdentifiableSource> linkedSources = derivedUnitBase2.getSources();
1335                 for (IdentifiableSource is:linkedSources){
1336                     Reference a = is.getCitation();
1337                     String c = is.getCitationMicroReference();
1338
1339                     boolean refMatch=false;
1340                     boolean microMatch=false;
1341
1342                     try{
1343                         if (a==null && b==null) {
1344                             refMatch=true;
1345                         }
1346                         if (a!=null && b!=null) {
1347                             if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1348                                 refMatch=true;
1349                             }
1350                         }
1351                     }catch(Exception e){}
1352
1353
1354                     try{
1355                         if (c==null && d==null) {
1356                             microMatch=true;
1357                         }
1358                         if(c!=null && d!=null) {
1359                             if(c.equalsIgnoreCase(d)) {
1360                                 microMatch=true;
1361                             }
1362                         }
1363                     }
1364                     catch(Exception e){}
1365
1366                     if (microMatch && refMatch) {
1367                         return false;
1368                     }
1369
1370
1371                 }
1372                 return true;
1373             }
1374
1375             private <T extends OriginalSourceBase<?>> boolean  sourceNotLinkedToElement(ISourceable<T> sourcable, Reference reference, String microReference) {
1376                 Set<T> linkedSources = sourcable.getSources();
1377                 for (T is:linkedSources){
1378                     Reference unitReference = is.getCitation();
1379                     String unitMicroReference = is.getCitationMicroReference();
1380
1381                     boolean refMatch=false;
1382                     boolean microMatch=false;
1383
1384                     try{
1385                         if (unitReference==null && reference==null) {
1386                             refMatch=true;
1387                         }
1388                         if (unitReference!=null && reference!=null) {
1389                             if (unitReference.getTitleCache().equalsIgnoreCase(reference.getTitleCache())) {
1390                                 refMatch=true;
1391                             }
1392                         }
1393                     }catch(Exception e){}
1394
1395                     try{
1396                         if (unitMicroReference==null && microReference==null) {
1397                             microMatch=true;
1398                         }
1399                         if(unitMicroReference!=null && microReference!=null) {
1400                             if(unitMicroReference.equalsIgnoreCase(microReference)) {
1401                                 microMatch=true;
1402                             }
1403                         }
1404                     }
1405                     catch(Exception e){}
1406
1407                     if (microMatch && refMatch) {
1408                         return false;
1409                     }
1410                 }
1411                 return true;
1412             }
1413
1414             /**
1415              * look for the Feature object (FieldObs, Specimen,...)
1416              * @param unit : a specimen or obersvation base
1417              * @return the corresponding Feature
1418              */
1419             private Feature makeFeature(SpecimenOrObservationBase<?> unit) {
1420                 SpecimenOrObservationType type = unit.getRecordBasis();
1421
1422
1423
1424                 if (type.isFeatureObservation()){
1425                     return Feature.OBSERVATION();
1426                 }else if (type.isFeatureSpecimen()){
1427                     return Feature.SPECIMEN();
1428                 }else if (type == SpecimenOrObservationType.DerivedUnit){
1429                     return Feature.OBSERVATION();
1430                     //            return getFeature("Specimen or observation");
1431                 }else{
1432                     String message = "Unhandled record basis '%s' for defining individuals association feature type. Use default.";
1433                     logger.warn(String.format(message, type.getMessage()));
1434                     return Feature.OBSERVATION();
1435                     //            return getFeature("Specimen or observation");
1436
1437                 }
1438             }
1439
1440
1441             /**
1442              * @param sourceMap
1443              * @param osbSet
1444              */
1445             protected void addToSourceMap(Map<String, OriginalSourceBase<?>> sourceMap, Set<OriginalSourceBase> osbSet) {
1446                 for( OriginalSourceBase<?> osb:osbSet) {
1447                     if(osb.getCitation()!=null && osb.getCitationMicroReference() !=null  && !osb.getCitationMicroReference().isEmpty()) {
1448                         try{
1449                             sourceMap.put(osb.getCitation().getTitleCache()+ "---"+osb.getCitationMicroReference(),osb);
1450                         }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1451                     } else if(osb.getCitation()!=null){
1452                         try{
1453                             sourceMap.put(osb.getCitation().getTitleCache(),osb);
1454                         }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1455                     }
1456                 }
1457             }
1458
1459
1460 }