cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/SpecimenImportBase.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.specimen;
  11
  12
  13 import java.util.ArrayList;
  14 import java.util.HashMap;
  15 import java.util.HashSet;
  16 import java.util.List;
  17 import java.util.Map;
  18 import java.util.Set;
  19 import java.util.UUID;
  20
  21 import org.apache.log4j.Logger;
  22
  23 import eu.etaxonomy.cdm.api.application.ICdmApplicationConfiguration;
  24 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
  25 import eu.etaxonomy.cdm.api.service.config.FindOccurrencesConfigurator;
  26 import eu.etaxonomy.cdm.api.service.pager.Pager;
  27 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  28 import eu.etaxonomy.cdm.io.common.CdmImportBase;
  29 import eu.etaxonomy.cdm.io.common.IImportConfigurator;
  30 import eu.etaxonomy.cdm.io.specimen.abcd206.in.Identification;
  31 import eu.etaxonomy.cdm.io.specimen.abcd206.in.SpecimenImportReport;
  32 import eu.etaxonomy.cdm.model.agent.AgentBase;
  33 import eu.etaxonomy.cdm.model.agent.Institution;
  34 import eu.etaxonomy.cdm.model.agent.Team;
  35 import eu.etaxonomy.cdm.model.common.CdmBase;
  36 import eu.etaxonomy.cdm.model.common.ISourceable;
  37 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  38 import eu.etaxonomy.cdm.model.common.LanguageString;
  39 import eu.etaxonomy.cdm.model.common.OriginalSourceBase;
  40 import eu.etaxonomy.cdm.model.common.OriginalSourceType;
  41 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  42 import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
  43 import eu.etaxonomy.cdm.model.description.Feature;
  44 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  45 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  46 import eu.etaxonomy.cdm.model.name.BacterialName;
  47 import eu.etaxonomy.cdm.model.name.BotanicalName;
  48 import eu.etaxonomy.cdm.model.name.CultivarPlantName;
  49 import eu.etaxonomy.cdm.model.name.INonViralName;
  50 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  51 import eu.etaxonomy.cdm.model.name.NonViralName;
  52 import eu.etaxonomy.cdm.model.name.Rank;
  53 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
  54 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
  55 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
  56 import eu.etaxonomy.cdm.model.name.ZoologicalName;
  57 import eu.etaxonomy.cdm.model.occurrence.Collection;
  58 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  59 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
  60 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
  61 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationType;
  62 import eu.etaxonomy.cdm.model.reference.Reference;
  63 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  64 import eu.etaxonomy.cdm.model.taxon.Classification;
  65 import eu.etaxonomy.cdm.model.taxon.Synonym;
  66 import eu.etaxonomy.cdm.model.taxon.Taxon;
  67 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  68 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  69 import eu.etaxonomy.cdm.persistence.dto.UuidAndTitleCache;
  70 import eu.etaxonomy.cdm.persistence.query.MatchMode;
  71 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
  72 import eu.etaxonomy.cdm.strategy.parser.ParserProblem;
  73
  74
  75 /**
  76  * @author p.kelbert
  77  * @created 20.10.2008
  78  */
  79 public abstract class SpecimenImportBase<CONFIG extends IImportConfigurator, STATE extends SpecimenImportStateBase>  extends CdmImportBase<CONFIG, STATE> {
  80
  81     private static final long serialVersionUID = 4423065367998125678L;
  82     private static final Logger logger = Logger.getLogger(SpecimenImportBase.class);
  83         protected final boolean DEBUG = true;
  84
  85         protected static final UUID SPECIMEN_SCAN_TERM = UUID.fromString("acda15be-c0e2-4ea8-8783-b9b0c4ad7f03");
  86
  87         private static final String COLON = ":";
  88
  89
  90         @Override
  91     protected abstract void doInvoke(STATE state);
  92
  93         /**
  94          * Handle a single unit
  95          * @param state
  96          * @param item
  97          */
  98         protected abstract void handleSingleUnit(STATE state, Object item) ;
  99
 100
 101
 102         protected TaxonNameBase<?, ?> getOrCreateTaxonName(String scientificName, Rank rank, boolean preferredFlag, STATE state, int unitIndexInAbcdFile){
 103         TaxonNameBase<?, ?> taxonName = null;
 104         SpecimenImportConfiguratorBase<?,?> config = state.getConfig();
 105
 106         //check atomised name data for rank
 107         //new name will be created
 108         NonViralName<?> atomisedTaxonName = null;
 109         if (rank==null && unitIndexInAbcdFile>=0 && ((state.getDataHolder().getAtomisedIdentificationList() != null && !state.getDataHolder().getAtomisedIdentificationList().isEmpty())|| state.getDataHolder().getAtomisedIdentificationList().size() > 0)) {
 110             atomisedTaxonName = setTaxonNameByType(state.getDataHolder().getAtomisedIdentificationList().get(unitIndexInAbcdFile), scientificName, state);
 111             if(atomisedTaxonName!=null){
 112                 rank = atomisedTaxonName.getRank();
 113             }
 114         }
 115         if(config.isReuseExistingTaxaWhenPossible()){
 116             NonViralName<?> parsedName = atomisedTaxonName;
 117             if(parsedName==null){
 118                 parsedName = parseScientificName(scientificName, state, state.getReport());
 119             }
 120             atomisedTaxonName = parsedName;
 121             if(config.isIgnoreAuthorship() && parsedName!=null && preferredFlag){
 122                 // do not ignore authorship for non-preferred names because they need
 123                 // to be created for the determination history
 124                 String nameCache = parsedName.getNameCache();
 125                 List<NonViralName> names = getNameService().findNamesByNameCache(nameCache, MatchMode.EXACT, null);
 126                 if (!names.isEmpty()){
 127                      return getBestMatchingName(scientificName, new ArrayList<TaxonNameBase>(names), state);
 128                 }
 129             } else {
 130                 //search for existing names
 131                 List<TaxonNameBase> names = getNameService().listByTitle(TaxonNameBase.class, scientificName, MatchMode.EXACT, null, null, null, null, null);
 132                 taxonName = getBestMatchingName(scientificName, names, state);
 133                 //still nothing found -> try with the atomised name full title cache
 134                 if(taxonName==null && atomisedTaxonName!=null){
 135                     names = getNameService().listByTitle(TaxonNameBase.class, atomisedTaxonName.getFullTitleCache(), MatchMode.EXACT, null, null, null, null, null);
 136                     taxonName = getBestMatchingName(atomisedTaxonName.getTitleCache(), names, state);
 137                     //still nothing found -> try with the atomised name title cache
 138                     if(taxonName==null){
 139                         names = getNameService().listByTitle(TaxonNameBase.class, atomisedTaxonName.getTitleCache(), MatchMode.EXACT, null, null, null, null, null);
 140                         taxonName = getBestMatchingName(atomisedTaxonName.getTitleCache(), names, state);
 141                     }
 142                 }
 143
 144             }
 145
 146         }
 147
 148         if(taxonName==null && atomisedTaxonName!=null){
 149             taxonName = atomisedTaxonName;
 150             state.getReport().addName(taxonName);
 151             logger.info("Created new taxon name "+taxonName);
 152             if(taxonName.hasProblem()){
 153                 state.getReport().addInfoMessage(String.format("Created %s with parsing problems", taxonName));
 154             }
 155             if(!atomisedTaxonName.getTitleCache().equals(scientificName)){
 156                 state.getReport().addInfoMessage(String.format("Taxon %s was parsed as %s", scientificName, atomisedTaxonName.getTitleCache()));
 157             }
 158         }
 159         else if(taxonName==null){
 160             //create new taxon name
 161
 162             if (state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNAFP)){
 163                 taxonName = BotanicalName.NewInstance(rank);
 164             }else if (state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICZN)){
 165                 taxonName = ZoologicalName.NewInstance(rank);
 166             }else{
 167                 taxonName = NonViralName.NewInstance(rank);
 168             }
 169             taxonName.setFullTitleCache(scientificName,true);
 170             taxonName.setTitleCache(scientificName, true);
 171             state.getReport().addName(taxonName);
 172             logger.info("Created new taxon name "+taxonName);
 173         }
 174         save(taxonName, state);
 175         return taxonName;
 176     }
 177
 178          protected TaxonNameBase<?, ?> getBestMatchingName(String scientificName, java.util.Collection<TaxonNameBase> names, STATE state){
 179                 List<TaxonNameBase> namesWithAcceptedTaxa = new ArrayList<>();
 180                 for (TaxonNameBase name : names) {
 181                     if(!name.getTaxa().isEmpty()){
 182                         namesWithAcceptedTaxa.add(name);
 183                     }
 184                 }
 185                 String message = "More than one taxon name was found for "+scientificName+"!";
 186                 //check for names with accepted taxa
 187                 if(namesWithAcceptedTaxa.size()>0){
 188                     if(namesWithAcceptedTaxa.size()>1){
 189                         state.getReport().addInfoMessage(message);
 190                         logger.warn(message);
 191                         return null;
 192                     }
 193                     return namesWithAcceptedTaxa.iterator().next();
 194                 }
 195                 //no names with accepted taxa found -> check accepted taxa of synonyms
 196                 List<Taxon> taxaFromSynonyms = new ArrayList<>();
 197                 for (TaxonNameBase name : names) {
 198                     Set<TaxonBase> taxonBases = name.getTaxonBases();
 199                     for (TaxonBase taxonBase : taxonBases) {
 200                         if(taxonBase.isInstanceOf(Synonym.class)){
 201                             Synonym synonym = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
 202                             taxaFromSynonyms.add(synonym.getAcceptedTaxon());
 203                         }
 204                     }
 205                 }
 206                 if(taxaFromSynonyms.size()>0){
 207                     if(taxaFromSynonyms.size()>1){
 208                         state.getReport().addInfoMessage(message);
 209                         logger.warn(message);
 210                         return null;
 211                     }
 212                     return taxaFromSynonyms.iterator().next().getName();
 213                 }
 214                 return null;
 215             }
 216          /**
 217              * Parse automatically the scientific name
 218              * @param scientificName the scientific name to parse
 219              * @param state the current import state
 220              * @param report the import report
 221              * @return a parsed name
 222              */
 223             protected NonViralName<?> parseScientificName(String scientificName, STATE state, SpecimenImportReport report) {
 224                 NonViralNameParserImpl nvnpi = NonViralNameParserImpl.NewInstance();
 225                 NonViralName<?> taxonName = null;
 226                 boolean problem = false;
 227
 228                 if(DEBUG){
 229                     logger.info("parseScientificName " + state.getDataHolder().getNomenclatureCode().toString());
 230                 }
 231
 232                 if (state.getDataHolder().getNomenclatureCode().toString().equals("Zoological") || state.getDataHolder().getNomenclatureCode().toString().contains("ICZN")) {
 233                     taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICZN, null);
 234                     if (taxonName.hasProblem()) {
 235                         problem = true;
 236                     }
 237                 }
 238                 else if (state.getDataHolder().getNomenclatureCode().toString().equals("Botanical") || state.getDataHolder().getNomenclatureCode().toString().contains("ICBN")) {
 239                     taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNAFP, null);
 240                     if (taxonName.hasProblem()) {
 241                         problem = true;
 242                     }
 243                 }
 244                 else if (state.getDataHolder().getNomenclatureCode().toString().equals("Bacterial") || state.getDataHolder().getNomenclatureCode().toString().contains("ICBN")) {
 245                     taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNB, null);
 246                     if (taxonName.hasProblem()) {
 247                         problem = true;
 248                     }
 249                 }
 250                 else if (state.getDataHolder().getNomenclatureCode().toString().equals("Cultivar") || state.getDataHolder().getNomenclatureCode().toString().contains("ICNCP")) {
 251                     taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNCP, null);
 252                     if (taxonName.hasProblem()) {
 253                         problem = true;
 254                     }
 255                 }
 256                 if (problem) {
 257                     String message = String.format("Parsing problems for %s", scientificName);
 258                     if(taxonName!=null){
 259                         for (ParserProblem parserProblem : taxonName.getParsingProblems()) {
 260                             message += "\n\t- "+parserProblem;
 261                         }
 262                     }
 263                     report.addInfoMessage(message);
 264                     logger.info(message);
 265                 }
 266                 return taxonName;
 267
 268             }
 269
 270             /**
 271              * Create the name without automatic parsing, either because it failed, or because the user deactivated it.
 272              * The name is built upon the ABCD fields
 273              * @param atomisedMap : the ABCD atomised fields
 274              * @param fullName : the full scientific name
 275              * @param state
 276              * @return the corresponding Botanical or Zoological or... name
 277              */
 278             protected NonViralName<?> setTaxonNameByType(
 279                     HashMap<String, String> atomisedMap, String fullName, STATE state) {
 280                 boolean problem = false;
 281                 if(DEBUG) {
 282                     logger.info("settaxonnamebytype " + state.getDataHolder().getNomenclatureCode().toString());
 283                 }
 284
 285                 if (state.getDataHolder().getNomenclatureCode().equals("Zoological") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICZN.getUuid())) {
 286                     NonViralName<ZoologicalName> taxonName = ZoologicalName.NewInstance(null);
 287                     taxonName.setFullTitleCache(fullName, true);
 288                     taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
 289                     taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
 290                     taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap,"SpeciesEpithet")));
 291                     taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap,"SubspeciesEpithet")));
 292
 293                     if (taxonName.getGenusOrUninomial() != null){
 294                         taxonName.setRank(Rank.GENUS());
 295                     }
 296
 297                     if (taxonName.getInfraGenericEpithet() != null){
 298                         taxonName.setRank(Rank.SUBGENUS());
 299                     }
 300
 301                     if (taxonName.getSpecificEpithet() != null){
 302                         taxonName.setRank(Rank.SPECIES());
 303                     }
 304
 305                     if (taxonName.getInfraSpecificEpithet() != null){
 306                         taxonName.setRank(Rank.SUBSPECIES());
 307                     }
 308
 309                     Team team = null;
 310                     if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 311                         team = Team.NewInstance();
 312                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
 313                     }
 314                     else {
 315                         if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 316                             team = Team.NewInstance();
 317                             team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
 318                         }
 319                     }
 320                     if (team != null) {
 321                         taxonName.setBasionymAuthorship(team);
 322                     }
 323                     else {
 324                         if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 325                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
 326                         }
 327                         else if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 328                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamAndYear"));
 329                         }
 330                     }
 331                     if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
 332                         team = Team.NewInstance();
 333                         team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
 334                         taxonName.setCombinationAuthorship(team);
 335                     }
 336                     if (taxonName.hasProblem()) {
 337                         logger.info("pb ICZN");
 338                         problem = true;
 339                     }
 340                     else {
 341                         return taxonName;
 342                     }
 343                 }
 344                 else if (state.getDataHolder().getNomenclatureCode().equals("Botanical") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNAFP.getUuid())) {
 345                     BotanicalName taxonName = (BotanicalName) parseScientificName(fullName, state, state.getReport());
 346                     if (taxonName != null){
 347                         return taxonName;
 348                     }
 349                     else{
 350                         taxonName = BotanicalName.NewInstance(null);
 351                     }
 352                     taxonName.setFullTitleCache(fullName, true);
 353                     taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
 354                     taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "FirstEpithet")));
 355                     taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "InfraSpeEpithet")));
 356                     try {
 357                         taxonName.setRank(Rank.getRankByName(getFromMap(atomisedMap, "Rank")));
 358                     } catch (Exception e) {
 359                         if (taxonName.getInfraSpecificEpithet() != null){
 360                             taxonName.setRank(Rank.SUBSPECIES());
 361                         }
 362                         else if (taxonName.getSpecificEpithet() != null){
 363                             taxonName.setRank(Rank.SPECIES());
 364                         }
 365                         else if (taxonName.getInfraGenericEpithet() != null){
 366                             taxonName.setRank(Rank.SUBGENUS());
 367                         }
 368                         else if (taxonName.getGenusOrUninomial() != null){
 369                             taxonName.setRank(Rank.GENUS());
 370                         }
 371                     }
 372                     Team team = null;
 373                     if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 374                         team = Team.NewInstance();
 375                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
 376                         taxonName.setBasionymAuthorship(team);
 377                     }
 378                     if (getFromMap(atomisedMap, "AuthorTeam") != null) {
 379                         team = Team.NewInstance();
 380                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeam"), true);
 381                         taxonName.setCombinationAuthorship(team);
 382                     }
 383                     if (team == null) {
 384                         if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 385                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
 386                         }
 387                         else if (getFromMap(atomisedMap, "AuthorTeam") != null) {
 388                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeam"));
 389                         }
 390                     }
 391                     if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
 392                         team = Team.NewInstance();
 393                         team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
 394                         taxonName.setCombinationAuthorship(team);
 395                     }
 396                     if (taxonName.hasProblem()) {
 397                         logger.info("pb ICBN");
 398                         problem = true;
 399                     }
 400                     else {
 401                         return taxonName;
 402                     }
 403                 }
 404                 else if (state.getDataHolder().getNomenclatureCode().equals("Bacterial") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNB.getUuid())) {
 405                     NonViralName<BacterialName> taxonName = BacterialName.NewInstance(null);
 406                     taxonName.setFullTitleCache(fullName, true);
 407                     taxonName.setGenusOrUninomial(getFromMap(atomisedMap, "Genus"));
 408                     taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
 409                     taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "Species")));
 410                     taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "SubspeciesEpithet")));
 411
 412                     if (taxonName.getGenusOrUninomial() != null){
 413                         taxonName.setRank(Rank.GENUS());
 414                     }
 415                     else if (taxonName.getInfraGenericEpithet() != null){
 416                         taxonName.setRank(Rank.SUBGENUS());
 417                     }
 418                     else if (taxonName.getSpecificEpithet() != null){
 419                         taxonName.setRank(Rank.SPECIES());
 420                     }
 421                     else if (taxonName.getInfraSpecificEpithet() != null){
 422                         taxonName.setRank(Rank.SUBSPECIES());
 423                     }
 424
 425                     if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 426                         Team team = Team.NewInstance();
 427                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
 428                         taxonName.setCombinationAuthorship(team);
 429                     }
 430                     if (getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear") != null) {
 431                         Team team = Team.NewInstance();
 432                         team.setTitleCache(getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear"), true);
 433                         taxonName.setBasionymAuthorship(team);
 434                     }
 435                     if (taxonName.hasProblem()) {
 436                         logger.info("pb ICNB");
 437                         problem = true;
 438                     }
 439                     else {
 440                         return taxonName;
 441                     }
 442                 }
 443                 else if (state.getDataHolder().getNomenclatureCode().equals("Cultivar")) {
 444                     CultivarPlantName taxonName = CultivarPlantName.NewInstance(null);
 445
 446                     if (taxonName.hasProblem()) {
 447                         logger.info("pb ICNCP");
 448                         problem = true;
 449                     }
 450                     else {
 451                         return taxonName;
 452                     }
 453                     return taxonName;
 454                 }
 455
 456                 if (problem) {
 457                     logger.info("Problem im setTaxonNameByType ");
 458                     NonViralName<?> taxonName = NonViralName.NewInstance(null);
 459                     taxonName.setFullTitleCache(fullName, true);
 460                     return taxonName;
 461                 }
 462                 NonViralName<?> tn = NonViralName.NewInstance(null);
 463                 return tn;
 464             }
 465
 466             /**
 467              * Get a formated string from a hashmap
 468              * @param atomisedMap
 469              * @param key
 470              * @return
 471              */
 472             private String getFromMap(HashMap<String, String> atomisedMap, String key) {
 473                 String value = null;
 474                 if (atomisedMap.containsKey(key)) {
 475                     value = atomisedMap.get(key);
 476                 }
 477
 478                 try {
 479                     if (value != null && key.matches(".*Year.*")) {
 480                         value = value.trim();
 481                         if (value.matches("[a-z A-Z ]*[0-9]{4}$")) {
 482                             String tmp = value.split("[0-9]{4}$")[0];
 483                             int year = Integer.parseInt(value.split(tmp)[1]);
 484                             if (year >= 1752) {
 485                                 value = tmp;
 486                             }
 487                             else {
 488                                 value = null;
 489                             }
 490                         }
 491                         else {
 492                             value = null;
 493                         }
 494                     }
 495                 }
 496                 catch (Exception e) {
 497                     value = null;
 498                 }
 499                 return value;
 500             }
 501
 502             /**
 503              * Very fast and dirty implementation to allow handling of transient objects as described in
 504              * https://dev.e-taxonomy.eu/trac/ticket/3726
 505              *
 506              * Not yet complete.
 507              *
 508              * @param cdmBase
 509              * @param state
 510              */
 511             protected UUID save(CdmBase cdmBase, SpecimenImportStateBase state) {
 512                 ICdmApplicationConfiguration cdmRepository = state.getConfig().getCdmAppController();
 513                 if (cdmRepository == null){
 514                     cdmRepository = this;
 515                 }
 516
 517                 if (cdmBase.isInstanceOf(LanguageString.class)){
 518                     return cdmRepository.getTermService().saveLanguageData(CdmBase.deproxy(cdmBase, LanguageString.class));
 519                 }else if (cdmBase.isInstanceOf(SpecimenOrObservationBase.class)){
 520                     return cdmRepository.getOccurrenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, SpecimenOrObservationBase.class));
 521                 }else if (cdmBase.isInstanceOf(Reference.class)){
 522                     return cdmRepository.getReferenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, Reference.class));
 523                 }else if (cdmBase.isInstanceOf(Classification.class)){
 524                     return cdmRepository.getClassificationService().saveOrUpdate(CdmBase.deproxy(cdmBase, Classification.class));
 525                 }else if (cdmBase.isInstanceOf(AgentBase.class)){
 526                     return cdmRepository.getAgentService().saveOrUpdate(CdmBase.deproxy(cdmBase, AgentBase.class));
 527                 }else if (cdmBase.isInstanceOf(Collection.class)){
 528                     return cdmRepository.getCollectionService().saveOrUpdate(CdmBase.deproxy(cdmBase, Collection.class));
 529                 }else if (cdmBase.isInstanceOf(DescriptionBase.class)){
 530                     return cdmRepository.getDescriptionService().saveOrUpdate(CdmBase.deproxy(cdmBase, DescriptionBase.class));
 531                 }else if (cdmBase.isInstanceOf(TaxonBase.class)){
 532                     return cdmRepository.getTaxonService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonBase.class));
 533                 }else if (cdmBase.isInstanceOf(TaxonNameBase.class)){
 534                     return cdmRepository.getNameService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonNameBase.class));
 535                 }else if (cdmBase.isInstanceOf(TaxonNode.class)){
 536                 return cdmRepository.getTaxonNodeService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonNode.class));
 537             }else{
 538                     throw new IllegalArgumentException("Class not supported in save method: " + CdmBase.deproxy(cdmBase, CdmBase.class).getClass().getSimpleName());
 539                 }
 540
 541             }
 542
 543
 544             protected SpecimenOrObservationBase findExistingSpecimen(String unitId, SpecimenImportStateBase state){
 545                 ICdmApplicationConfiguration cdmAppController = state.getConfig().getCdmAppController();
 546                 if(cdmAppController==null){
 547                     cdmAppController = this;
 548                 }
 549                 FindOccurrencesConfigurator config = new FindOccurrencesConfigurator();
 550                 config.setSignificantIdentifier(unitId);
 551                 Pager<SpecimenOrObservationBase> existingSpecimens = cdmAppController.getOccurrenceService().findByTitle(config);
 552                 if(!existingSpecimens.getRecords().isEmpty()){
 553                     if(existingSpecimens.getRecords().size()==1){
 554                         return existingSpecimens.getRecords().iterator().next();
 555                     }
 556                 }
 557                 return null;
 558             }
 559
 560             protected abstract void importAssociatedUnits(STATE state, Object item, DerivedUnitFacade derivedUnitFacade);
 561
 562             /**
 563              * getFacade : get the DerivedUnitFacade based on the recordBasis
 564              * @param state
 565              *
 566              * @return DerivedUnitFacade
 567              */
 568             protected DerivedUnitFacade getFacade(STATE state) {
 569                 if(DEBUG) {
 570                     logger.info("getFacade()");
 571                 }
 572                 SpecimenOrObservationType type = null;
 573
 574                 // create specimen
 575                 if (NB((state.getDataHolder().getRecordBasis())) != null) {
 576                     if (state.getDataHolder().getRecordBasis().toLowerCase().startsWith("s") || state.getDataHolder().getRecordBasis().toLowerCase().indexOf("specimen")>-1) {// specimen
 577                         type = SpecimenOrObservationType.PreservedSpecimen;
 578                     }
 579                     if (state.getDataHolder().getRecordBasis().toLowerCase().startsWith("o") ||state.getDataHolder().getRecordBasis().toLowerCase().indexOf("observation")>-1 ) {
 580                         type = SpecimenOrObservationType.Observation;
 581                     }
 582                     if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("fossil")>-1){
 583                         type = SpecimenOrObservationType.Fossil;
 584                     }
 585                     if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("living")>-1) {
 586                         type = SpecimenOrObservationType.LivingSpecimen;
 587                     }
 588                     if (type == null) {
 589                         logger.info("The basis of record does not seem to be known: " + state.getDataHolder().getRecordBasis());
 590                         type = SpecimenOrObservationType.DerivedUnit;
 591                     }
 592                     // TODO fossils?
 593                 } else {
 594                     logger.info("The basis of record is null");
 595                     type = SpecimenOrObservationType.DerivedUnit;
 596                 }
 597                 DerivedUnitFacade derivedUnitFacade = DerivedUnitFacade.NewInstance(type);
 598                 return derivedUnitFacade;
 599             }
 600
 601             /**
 602              * Look if the Institution does already exist
 603              * @param institutionCode: a string with the institutioncode
 604              * @param config : the configurator
 605              * @return the Institution (existing or new)
 606              */
 607             protected Institution getInstitution(String institutionCode, STATE state) {
 608                 SpecimenImportConfiguratorBase config = state.getConfig();
 609                 Institution institution=null;
 610                 institution = (Institution)state.institutions.get(institutionCode);
 611                 if (institution != null){
 612                     return institution;
 613                 }
 614                 List<Institution> institutions;
 615                 try {
 616                     institutions = getAgentService().searchInstitutionByCode(institutionCode);
 617
 618                 } catch (Exception e) {
 619                     institutions = new ArrayList<Institution>();
 620                     logger.warn(e);
 621                 }
 622                 if (institutions.size() > 0 && config.isReuseExistingMetaData()) {
 623                     for (Institution institut:institutions){
 624                         try{
 625                             if (institut.getCode().equalsIgnoreCase(institutionCode)) {
 626                                 institution=institut;
 627                                 break;
 628                             }
 629                         }catch(Exception e){logger.warn("no institution code in the db");}
 630                     }
 631                 }
 632                 if(DEBUG) {
 633                     if(institution !=null) {
 634                         logger.info("getinstitution " + institution.toString());
 635                     }
 636                 }
 637                 if (institution == null){
 638                     // create institution
 639                     institution = Institution.NewInstance();
 640                     institution.setCode(institutionCode);
 641                     institution.setTitleCache(institutionCode, true);
 642                     UUID uuid = save(institution, state);
 643                 }
 644
 645
 646                 state.institutions.put(institutionCode, institution);
 647                 return institution;
 648             }
 649
 650             /**
 651              * Look if the Collection does already exist
 652              * @param collectionCode
 653              * @param collectionCode: a string
 654              * @param config : the configurator
 655              * @return the Collection (existing or new)
 656              */
 657             protected Collection getCollection(Institution institution, String collectionCode, STATE state) {
 658                 SpecimenImportConfiguratorBase config = state.getConfig();
 659                 Collection collection = null;
 660                 List<Collection> collections;
 661                 collection = (Collection) state.collections.get(collectionCode);
 662                 if (collection != null){
 663                     return collection;
 664                 }
 665                 try {
 666                     collections = getCollectionService().searchByCode(collectionCode);
 667                 } catch (Exception e) {
 668                     collections = new ArrayList<Collection>();
 669                 }
 670                 if (collections.size() > 0 && config.isReuseExistingMetaData()) {
 671                     for (Collection coll:collections){
 672                         if (coll.getCode() != null && coll.getInstitute() != null
 673                                 && coll.getCode().equalsIgnoreCase(collectionCode) && coll.getInstitute().equals(institution)) {
 674                             collection = coll;
 675                             break;
 676                         }
 677                     }
 678                 }
 679
 680                 if(collection == null){
 681                     collection =Collection.NewInstance();
 682                     collection.setCode(collectionCode);
 683                     collection.setInstitute(institution);
 684                     collection.setTitleCache(collectionCode);
 685                     UUID uuid = save(collection, state);
 686                 }
 687
 688
 689
 690                 state.collections.put(collectionCode, collection);
 691
 692                 return collection;
 693             }
 694
 695             /**
 696              * @param reference
 697              * @param citationDetail
 698              * @return
 699              */
 700             //FIXME this method is highly critical, because
 701             //  * it will have serious performance and memory problems with large databases
 702             //        (databases may easily have >1 Mio source records)
 703             //  * it does not make sense to search for existing sources and then clone them
 704             //    we need to search for existing references instead and use them (if exist)
 705             //    for our new source.
 706             protected IdentifiableSource getIdentifiableSource(Reference reference, String citationDetail) {
 707
 708               /*  List<IdentifiableSource> issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
 709
 710
 711                 if (reference != null){
 712                     try {
 713                         for (OriginalSourceBase<?> osb: issTmp){
 714                             if (osb.getCitation() != null && osb.getCitation().getTitleCache().equalsIgnoreCase(reference.getTitleCache())){
 715                                 String osbDetail = osb.getCitationMicroReference();
 716                                 if ((StringUtils.isBlank(osbDetail) && StringUtils.isBlank(citationDetail))
 717                                         || (osbDetail != null && osbDetail.equalsIgnoreCase(citationDetail)) ) {
 718 //                                  System.out.println("REFERENCE FOUND RETURN EXISTING SOURCE");
 719                                     return (IdentifiableSource) osb.clone();
 720                                 }
 721                             }
 722                         }
 723                     } catch (CloneNotSupportedException e) {
 724                         throw new RuntimeException(e);
 725                     } catch (Exception e1){
 726                         e1.printStackTrace();
 727                     }
 728                 }
 729         */
 730                 IdentifiableSource sour = IdentifiableSource.NewInstance(OriginalSourceType.Import,null,null, reference,citationDetail);
 731                 return sour;
 732             }
 733
 734             /**
 735              * Add the hierarchy for a Taxon(add higher taxa)
 736              * @param classification
 737              * @param taxon: a taxon to add as a node
 738              * @param state: the ABCD import state
 739              */
 740             protected void addParentTaxon(Taxon taxon, STATE state, boolean preferredFlag, Classification classification){
 741                 INonViralName<?>  nvname = taxon.getName();
 742                 Rank rank = nvname.getRank();
 743                 Taxon genus =null;
 744                 Taxon subgenus =null;
 745                 Taxon species = null;
 746                 Taxon subspecies = null;
 747                 Taxon parent = null;
 748                 if(rank!=null){
 749                     if (rank.isLower(Rank.GENUS() )){
 750                         String genusOrUninomial = nvname.getGenusOrUninomial();
 751                         TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(genusOrUninomial, Rank.GENUS(), preferredFlag, state, -1);
 752                         genus = getOrCreateTaxonForName(taxonName, state);
 753                         if (preferredFlag) {
 754                             parent = linkParentChildNode(null, genus, classification, state);
 755                         }
 756
 757                     }
 758                     if (rank.isLower(Rank.SUBGENUS())){
 759                         String prefix = nvname.getGenusOrUninomial();
 760                         String name = nvname.getInfraGenericEpithet();
 761                         if (name != null){
 762                             TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(prefix+" "+name, Rank.SUBGENUS(), preferredFlag, state, -1);
 763                             subgenus = getOrCreateTaxonForName(taxonName, state);
 764                             if (preferredFlag) {
 765                                 parent = linkParentChildNode(genus, subgenus, classification, state);
 766                             }            }
 767                     }
 768                     if (rank.isLower(Rank.SPECIES())){
 769                         if (subgenus!=null){
 770                             String prefix = nvname.getGenusOrUninomial();
 771                             String name = nvname.getInfraGenericEpithet();
 772                             String spe = nvname.getSpecificEpithet();
 773                             if (spe != null){
 774                                 TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(prefix+" "+name+" "+spe, Rank.SPECIES(), preferredFlag, state, -1);
 775                                 species = getOrCreateTaxonForName(taxonName, state);
 776                                 if (preferredFlag) {
 777                                     parent = linkParentChildNode(subgenus, species, classification, state);
 778                                 }
 779                             }
 780                         }
 781                         else{
 782                             String prefix = nvname.getGenusOrUninomial();
 783                             String name = nvname.getSpecificEpithet();
 784                             if (name != null){
 785                                 TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(prefix+" "+name, Rank.SPECIES(), preferredFlag, state, -1);
 786                                 species = getOrCreateTaxonForName(taxonName, state);
 787                                 if (preferredFlag) {
 788                                     parent = linkParentChildNode(genus, species, classification, state);
 789                                 }
 790                             }
 791                         }
 792                     }
 793                     if (rank.isLower(Rank.INFRASPECIES())){
 794                         TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(nvname.getFullTitleCache(), Rank.SUBSPECIES(), preferredFlag, state, -1);
 795                         subspecies = getOrCreateTaxonForName(taxonName, state);
 796                         if (preferredFlag) {
 797                             parent = linkParentChildNode(species, subspecies, classification, state);
 798                         }
 799                     }
 800                 }
 801                 if (preferredFlag && parent!=taxon) {
 802                     linkParentChildNode(parent, taxon, classification, state);
 803                 }
 804             }
 805
 806             /**
 807              * Link a parent to a child and save it in the current classification
 808              * @param parent: the higher Taxon
 809              * @param child : the lower (or current) Taxon
 810              * return the Taxon from the new created Node
 811              * @param classification
 812              * @param state
 813              */
 814             protected Taxon linkParentChildNode(Taxon parent, Taxon child, Classification classification, STATE state) {
 815                 TaxonNode node =null;
 816                 if (parent != null) {
 817                     parent = (Taxon) getTaxonService().find(parent.getUuid());
 818                     child = (Taxon) getTaxonService().find(child.getUuid());
 819                     //here we do not have to check if the taxon nodes already exists
 820                     //this is done by classification.addParentChild()
 821                     //do not add child node if it already exists
 822                     if(hasTaxonNodeInClassification(child, classification)){
 823                         return child;
 824                     }
 825                     else{
 826                         node = classification.addParentChild(parent, child, state.getRef(), "");
 827                         save(node, state);
 828                     }
 829                 }
 830                 else {
 831                     child = (Taxon) getTaxonService().find(child.getUuid());
 832                     //do not add child node if it already exists
 833                     if(hasTaxonNodeInClassification(child, classification)){
 834                         return child;
 835                     }
 836                     else{
 837                         node = classification.addChildTaxon(child, state.getRef(), null);
 838                         save(node, state);
 839                     }
 840                 }
 841                 if(node!=null){
 842                     state.getReport().addTaxonNode(node);
 843                     return node.getTaxon();
 844                 }
 845                 String message = "Could not create taxon node for " +child;
 846                 state.getReport().addInfoMessage(message);
 847                 logger.warn(message);
 848                 return null;
 849             }
 850
 851             protected Taxon getOrCreateTaxonForName(TaxonNameBase<?, ?> taxonNameBase, STATE state){
 852
 853                 Set<Taxon> acceptedTaxa = taxonNameBase.getTaxa();
 854                 if(acceptedTaxa.size()>0){
 855                     Taxon firstAcceptedTaxon = acceptedTaxa.iterator().next();
 856                     if(acceptedTaxa.size()>1){
 857                         String message = "More than one accepted taxon was found for taxon name: "
 858                                 + taxonNameBase.getTitleCache() + "!\n" + firstAcceptedTaxon + "was chosen for "+state.getDerivedUnitBase();
 859                         state.getReport().addInfoMessage(message);
 860                         logger.warn(message);
 861                     }
 862                     else{
 863                         return firstAcceptedTaxon;
 864                     }
 865                 }
 866                 else{
 867                     Set<TaxonBase> taxonAndSynonyms = taxonNameBase.getTaxonBases();
 868                     for (TaxonBase taxonBase : taxonAndSynonyms) {
 869                         if(taxonBase.isInstanceOf(Synonym.class)){
 870                             Synonym synonym = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
 871                             Taxon acceptedTaxonOfSynonym = synonym.getAcceptedTaxon();
 872                             if(acceptedTaxonOfSynonym == null){
 873                                 String message = "No accepted taxon could be found for taxon name: "
 874                                         + taxonNameBase.getTitleCache()
 875                                         + "!";
 876                                 state.getReport().addInfoMessage(message);
 877                                 logger.warn(message);
 878                             }
 879                             else{
 880                                 return acceptedTaxonOfSynonym;
 881                             }
 882                         }
 883                     }
 884                 }
 885                 Taxon taxon = Taxon.NewInstance(taxonNameBase, state.getRef());
 886                 save(taxon, state);
 887                 state.getReport().addTaxon(taxon);
 888                 logger.info("Created new taxon "+ taxon);
 889                 return taxon;
 890             }
 891
 892             private boolean hasTaxonNodeInClassification(Taxon taxon, Classification classification){
 893                 if(taxon.getTaxonNodes()!=null){
 894                     for (TaxonNode node : taxon.getTaxonNodes()){
 895                         if(node.getClassification().equals(classification)){
 896                             return true;
 897                         }
 898                     }
 899                 }
 900                 return false;
 901             }
 902
 903             /**
 904              * HandleIdentifications : get the scientific names present in the ABCD
 905              * document and store link them with the observation/specimen data
 906              * @param state: the current ABCD import state
 907              * @param derivedUnitFacade : the current derivedunitfacade
 908              */
 909             protected void handleIdentifications(STATE state, DerivedUnitFacade derivedUnitFacade) {
 910                SpecimenImportConfiguratorBase config = state.getConfig();
 911
 912
 913                 String scientificName = "";
 914                 boolean preferredFlag = false;
 915
 916                 if (state.getDataHolder().getNomenclatureCode() == ""){
 917                     if (config.getNomenclaturalCode() != null){
 918                         if (config.getNomenclaturalCode() != null){
 919                             state.getDataHolder().setNomenclatureCode(config.getNomenclaturalCode().toString());
 920                         }
 921                     }
 922                 }
 923
 924                 for (int i = 0; i < state.getDataHolder().getIdentificationList().size(); i++) {
 925                     Identification identification = state.getDataHolder().getIdentificationList().get(i);
 926                     scientificName = identification.getScientificName().replaceAll(" et ", " & ");
 927
 928                     String preferred = identification.getPreferred();
 929                     if (preferred.equals("1") || preferred.toLowerCase().indexOf("true") != -1 || state.getDataHolder().getIdentificationList().size()==1) {
 930                         preferredFlag = true;
 931                     }
 932                     else {
 933                         preferredFlag = false;
 934                     }
 935                     if (identification.getCode() != null){
 936                     if (identification.getCode().indexOf(':') != -1) {
 937                         state.getDataHolder().setNomenclatureCode(identification.getCode().split(COLON)[1]);
 938                     }
 939                     else{
 940                         state.getDataHolder().setNomenclatureCode(identification.getCode());
 941                     }
 942                     }
 943                     TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(scientificName, null, preferredFlag, state, i);
 944                     Taxon taxon = getOrCreateTaxonForName(taxonName, state);
 945                     addTaxonNode(taxon, state,preferredFlag);
 946                     linkDeterminationEvent(state, taxon, preferredFlag, derivedUnitFacade);
 947                 }
 948             }
 949
 950             /**
 951              * @param taxon : a taxon to add as a node
 952              * @param state : the ABCD import state
 953              */
 954             protected void addTaxonNode(Taxon taxon, STATE state, boolean preferredFlag) {
 955                 SpecimenImportConfiguratorBase config = state.getConfig();
 956                 logger.info("link taxon to a taxonNode "+taxon.getTitleCache());
 957                 //only add nodes if not already existing in current classification or default classification
 958
 959                 //check if node exists in current classification
 960                 //NOTE: we cannot use hasTaxonNodeInClassification() here because we are first creating it here
 961                 if (!existsInClassification(taxon,state.getClassification(), state)){
 962                     if(config.isMoveNewTaxaToDefaultClassification()){
 963                         //check if node exists in default classification
 964                         if (!existsInClassification(taxon, state.getDefaultClassification(), state)){
 965                             addParentTaxon(taxon, state, preferredFlag, state.getDefaultClassification());
 966                         }
 967                     }
 968                     else {
 969                         //add non-existing taxon to current classification
 970                         addParentTaxon(taxon, state, preferredFlag, state.getClassification());
 971                     }
 972                 }
 973             }
 974
 975
 976             private boolean existsInClassification(Taxon taxon, Classification classification, STATE state){
 977                 boolean exist = false;
 978                 ICdmApplicationConfiguration cdmAppController = state.getConfig().getCdmAppController();
 979             if(cdmAppController==null){
 980                 cdmAppController = this;
 981             }
 982             if (classification != null){
 983                 List<UuidAndTitleCache<TaxonNode>> uuidAndTitleCacheOfAllTaxa = cdmAppController.getClassificationService().getTaxonNodeUuidAndTitleCacheOfAcceptedTaxaByClassification(classification.getUuid());
 984                 if (uuidAndTitleCacheOfAllTaxa != null){
 985                         for (UuidAndTitleCache p : uuidAndTitleCacheOfAllTaxa){
 986                             try{
 987                                 if(p.getTitleCache().equals(taxon.getTitleCache())) {
 988                                     exist = true;
 989                                 }
 990                             }
 991                             catch(Exception e){
 992                                 logger.warn("TaxonNode doesn't seem to have a taxon");
 993                             }
 994                         }
 995                 }
 996             }
 997                 return exist;
 998             }
 999
1000             /**
1001              * join DeterminationEvent to the Taxon Object
1002              * @param state : the ABCD import state
1003              * @param taxon: the current Taxon
1004              * @param preferredFlag :if the current name is preferred
1005              * @param derivedFacade : the derived Unit Facade
1006              */
1007             @SuppressWarnings("rawtypes")
1008         protected void linkDeterminationEvent(STATE state, Taxon taxon, boolean preferredFlag,  DerivedUnitFacade derivedFacade) {
1009                 SpecimenImportConfiguratorBase config = state.getConfig();
1010                 if(DEBUG){
1011                     logger.info("start linkdetermination with taxon:" + taxon.getUuid()+", "+taxon);
1012                 }
1013
1014                 DeterminationEvent determinationEvent = DeterminationEvent.NewInstance();
1015                 determinationEvent.setTaxonName(taxon.getName());
1016                 determinationEvent.setPreferredFlag(preferredFlag);
1017
1018                 determinationEvent.setIdentifiedUnit(state.getDerivedUnitBase());
1019                 state.getDerivedUnitBase().addDetermination(determinationEvent);
1020
1021                 if(DEBUG){
1022                     logger.info("NB TYPES INFO: "+ state.getDataHolder().getStatusList().size());
1023                 }
1024                 for (SpecimenTypeDesignationStatus specimenTypeDesignationstatus : state.getDataHolder().getStatusList()) {
1025                     if (specimenTypeDesignationstatus != null) {
1026                         if(DEBUG){
1027                             logger.info("specimenTypeDesignationstatus :"+ specimenTypeDesignationstatus);
1028                         }
1029
1030                         ICdmApplicationConfiguration cdmAppController = config.getCdmAppController();
1031                         if(cdmAppController == null){
1032                             cdmAppController = this;
1033                         }
1034                         specimenTypeDesignationstatus = (SpecimenTypeDesignationStatus) cdmAppController.getTermService().find(specimenTypeDesignationstatus.getUuid());
1035                         //Designation
1036                         TaxonNameBase<?,?> name = taxon.getName();
1037                         SpecimenTypeDesignation designation = SpecimenTypeDesignation.NewInstance();
1038
1039                         designation.setTypeStatus(specimenTypeDesignationstatus);
1040                         designation.setTypeSpecimen(state.getDerivedUnitBase());
1041                         name.addTypeDesignation(designation, true);
1042                     }
1043                 }
1044
1045                 for (String[] fullReference : state.getDataHolder().getReferenceList()) {
1046
1047
1048                     String strReference=fullReference[0];
1049                     String citationDetail = fullReference[1];
1050                     String citationURL = fullReference[2];
1051                     List<Reference> references = getReferenceService().listByTitle(Reference.class, "strReference", MatchMode.EXACT, null, null, null, null, null);
1052
1053                     if (!references.isEmpty()){
1054                         Reference reference = null;
1055                         for (Reference refe: references) {
1056                             if (refe.getTitleCache().equalsIgnoreCase(strReference)) {
1057                                 reference =refe;
1058                                 break;
1059                             }
1060                         }
1061                         if (reference ==null){
1062                             reference = ReferenceFactory.newGeneric();
1063                             reference.setTitleCache(strReference, true);
1064                             save(reference, state);
1065                         }
1066                         determinationEvent.addReference(reference);
1067                     }
1068                 }
1069                 save(state.getDerivedUnitBase(), state);
1070
1071                 if (config.isAddIndividualsAssociationsSuchAsSpecimenAndObservations() && preferredFlag) {
1072                     //do not add IndividualsAssociation to non-preferred taxa
1073                     if(DEBUG){
1074                         logger.info("isDoCreateIndividualsAssociations");
1075                     }
1076
1077                     makeIndividualsAssociation(state, taxon, determinationEvent);
1078
1079                     save(state.getDerivedUnitBase(), state);
1080                 }
1081             }
1082
1083             /**
1084              * create and link each association (specimen, observation..) to the accepted taxon
1085              * @param state : the ABCD import state
1086              * @param taxon: the current Taxon
1087              * @param determinationEvent:the determinationevent
1088              */
1089             protected void makeIndividualsAssociation(STATE state, Taxon taxon, DeterminationEvent determinationEvent) {
1090                 SpecimenImportConfiguratorBase config = state.getConfig();
1091                 SpecimenUserInteraction sui = config.getSpecimenUserInteraction();
1092
1093                 if (DEBUG) {
1094                     logger.info("MAKE INDIVIDUALS ASSOCIATION");
1095                 }
1096
1097                 TaxonDescription taxonDescription = null;
1098                 Set<TaxonDescription> descriptions= taxon.getDescriptions();
1099 //              if (((Abcd206ImportConfigurator) state.getConfig()).isInteractWithUser()){
1100 //                  if(!state.isDescriptionGroupSet()){
1101 //                      taxonDescription = sui.askForDescriptionGroup(descriptions);
1102 //                      state.setDescriptionGroup(taxonDescription);
1103 //                      state.setDescriptionGroupSet(true);
1104 //                  }else{
1105 //                      taxonDescription=state.getDescriptionGroup();
1106 //                  }
1107 //              } else {
1108                     for (TaxonDescription description : descriptions){
1109                         Set<IdentifiableSource> sources =  new HashSet<>();
1110                         sources.addAll(description.getTaxon().getSources());
1111                         sources.addAll(description.getSources());
1112                         for (IdentifiableSource source:sources){
1113                             if(state.getRef().equals(source.getCitation())) {
1114                                 taxonDescription = description;
1115                             }
1116                         }
1117                     }
1118               //  }
1119                 if (taxonDescription == null){
1120                     taxonDescription = TaxonDescription.NewInstance(taxon, false);
1121                     if(sourceNotLinkedToElement(taxonDescription,state.getRef(),null)) {
1122                         taxonDescription.addSource(OriginalSourceType.Import, null, null, state.getRef(), null);
1123                     }
1124                     state.setDescriptionGroup(taxonDescription);
1125                     taxon.addDescription(taxonDescription);
1126                 }
1127
1128                 //PREPARE REFERENCE QUESTIONS
1129
1130                 Map<String,OriginalSourceBase<?>> sourceMap = new HashMap<String, OriginalSourceBase<?>>();
1131
1132                 List<IdentifiableSource> issTmp = new ArrayList<>();//getCommonService().list(IdentifiableSource.class, null, null, null, null);
1133                 List<DescriptionElementSource> issTmp2 = new ArrayList<>();//getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1134
1135                 Set<OriginalSourceBase> osbSet = new HashSet<OriginalSourceBase>();
1136                 if(issTmp2!=null) {
1137                     osbSet.addAll(issTmp2);
1138                 }
1139                 if(issTmp!=null) {
1140                     osbSet.addAll(issTmp);
1141                 }
1142
1143
1144                 addToSourceMap(sourceMap, osbSet);
1145
1146 //              if (((Abcd206ImportConfigurator) state.getConfig()).isInteractWithUser()){
1147 //                  List<OriginalSourceBase<?>> res = null;
1148 //                  if(!state.isDescriptionSourcesSet()){
1149 //                      res = sui.askForSource(sourceMap, "the description group ("+taxon+")",
1150 //                              "The current reference is "+state.getRef().getTitleCache(),getReferenceService(), state.getDataHolder().docSources);
1151 //                      state.setDescriptionRefs(res);
1152 //                      state.setDescriptionSourcesSet(true);
1153 //                  }
1154 //                  else{
1155 //                      res=state.getDescriptionRefs();
1156 //                  }
1157 //                  if(res !=null) {
1158 //                      for (OriginalSourceBase<?> sour:res){
1159 //                          if(sour.isInstanceOf(IdentifiableSource.class)){
1160 //                              try {
1161 //                                  if(sourceNotLinkedToElement(taxonDescription,sour)) {
1162 //                                      taxonDescription.addSource((IdentifiableSource)sour.clone());
1163 //                                  }
1164 //                              } catch (CloneNotSupportedException e) {
1165 //                                  logger.warn("no cloning?");
1166 //                              }
1167 //                          }else{
1168 //                              if(sourceNotLinkedToElement(taxonDescription,sour)) {
1169 //                                  taxonDescription.addSource(OriginalSourceType.Import,null, null, sour.getCitation(),sour.getCitationMicroReference());
1170 //                              }
1171 //                          }
1172 //                      }
1173 //                  }
1174 //              }
1175 //              else {
1176                     if(sourceNotLinkedToElement(taxonDescription,state.getRef(),null)) {
1177                         taxonDescription.addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1178                     }
1179 //              }
1180                 state.setDescriptionGroup(taxonDescription);
1181
1182                 IndividualsAssociation indAssociation = IndividualsAssociation.NewInstance();
1183                 Feature feature = makeFeature(state.getDerivedUnitBase());
1184                 indAssociation.setAssociatedSpecimenOrObservation(state.getDerivedUnitBase());
1185                 indAssociation.setFeature(feature);
1186 //
1187 //              if (((Abcd206ImportConfigurator) state.getConfig()).isInteractWithUser()){
1188 //                  sourceMap = new HashMap<String, OriginalSourceBase<?>>();
1189 //
1190 //                  issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
1191 //                  issTmp2 = getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1192 //
1193 //                  osbSet = new HashSet<OriginalSourceBase>();
1194 //                  if(issTmp2!=null) {
1195 //                      osbSet.addAll(issTmp2);
1196 //                  }
1197 //                  if(issTmp!=null) {
1198 //                      osbSet.addAll(issTmp);
1199 //                  }
1200 //
1201 //
1202 //                  addToSourceMap(sourceMap, osbSet);
1203 //
1204 //                  List<OriginalSourceBase<?>> sources =null;
1205 //                  if(!state.isAssociationSourcesSet()) {
1206 //                      sources = sui.askForSource(sourceMap,  "descriptive element (association) ",taxon.toString(),
1207 //                              getReferenceService(),state.getDataHolder().getDocSources());
1208 //                      state.setAssociationRefs(sources);
1209 //                      state.setAssociationSourcesSet(true);
1210 //                  }
1211 //                  else{
1212 //                      sources=state.getAssociationRefs();
1213 //                  }
1214 //                  if(sources !=null) {
1215 //                      for (OriginalSourceBase<?> source: sources) {
1216 //                          if(source !=null) {
1217 //                              if(source.isInstanceOf(DescriptionElementSource.class)){
1218 //                                  try {
1219 //                                      if(sourceNotLinkedToElement(indAssociation,source)) {
1220 //                                          indAssociation.addSource((DescriptionElementSource)source.clone());
1221 //                                      }
1222 //                                  } catch (CloneNotSupportedException e) {
1223 //                                      logger.warn("clone forbidden?");
1224 //                                  }
1225 //                              }else{
1226 //                                  if(sourceNotLinkedToElement(indAssociation,source)) {
1227 //                                      indAssociation.addSource(OriginalSourceType.Import,null, null, source.getCitation(),source.getCitationMicroReference());
1228 //                                  }
1229 //                                  try {
1230 //                                      if(sourceNotLinkedToElement(state.getDerivedUnitBase(), source)) {
1231 //                                          state.getDerivedUnitBase().addSource((IdentifiableSource) source.clone());
1232 //                                      }
1233 //                                  } catch (CloneNotSupportedException e) {
1234 //                                      // TODO Auto-generated catch block
1235 //                                      e.printStackTrace();
1236 //                                  }
1237 //                              }
1238 //
1239 //                          }
1240 //                      }
1241 //                  }
1242 //              }else {
1243                     if(sourceNotLinkedToElement(indAssociation,state.getRef(),null)) {
1244                         indAssociation.addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1245                     }
1246                     if(sourceNotLinkedToElement(state.getDerivedUnitBase(), state.getRef(),null)) {
1247                         state.getDerivedUnitBase().addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1248                     }
1249                     for (Reference citation : determinationEvent.getReferences()) {
1250                         if(sourceNotLinkedToElement(indAssociation,citation,null))
1251                         {
1252                             indAssociation.addSource(DescriptionElementSource.NewInstance(OriginalSourceType.Import, null, null, citation, null));
1253                         }
1254                         if(sourceNotLinkedToElement(state.getDerivedUnitBase(), state.getRef(),null)) {
1255                             state.getDerivedUnitBase().addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1256                         }
1257                     }
1258            //     }
1259
1260                 taxonDescription.addElement(indAssociation);
1261
1262                 save(taxonDescription, state);
1263                 save(taxon, state);
1264                 state.getReport().addDerivate(state.getDerivedUnitBase(), config);
1265                 state.getReport().addIndividualAssociation(taxon, state.getDataHolder().getUnitID(), state.getDerivedUnitBase());
1266             }
1267
1268             /**
1269              * @param derivedUnitBase2
1270              * @param ref2
1271              * @param object
1272              * @return
1273              */
1274             private boolean sourceNotLinkedToElement(DerivedUnit derivedUnitBase2, Reference b, String d) {
1275                 Set<IdentifiableSource> linkedSources = derivedUnitBase2.getSources();
1276                 for (IdentifiableSource is:linkedSources){
1277                     Reference a = is.getCitation();
1278                     String c = is.getCitationMicroReference();
1279
1280                     boolean refMatch=false;
1281                     boolean microMatch=false;
1282
1283                     try{
1284                         if (a==null && b==null) {
1285                             refMatch=true;
1286                         }
1287                         if (a!=null && b!=null) {
1288                             if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1289                                 refMatch=true;
1290                             }
1291                         }
1292                     }catch(Exception e){}
1293
1294
1295                     try{
1296                         if (c==null && d==null) {
1297                             microMatch=true;
1298                         }
1299                         if(c!=null && d!=null) {
1300                             if(c.equalsIgnoreCase(d)) {
1301                                 microMatch=true;
1302                             }
1303                         }
1304                     }
1305                     catch(Exception e){}
1306
1307                     if (microMatch && refMatch) {
1308                         return false;
1309                     }
1310
1311
1312                 }
1313                 return true;
1314             }
1315
1316             private <T extends OriginalSourceBase<?>> boolean  sourceNotLinkedToElement(ISourceable<T> sourcable, Reference reference, String microReference) {
1317                 Set<T> linkedSources = sourcable.getSources();
1318                 for (T is:linkedSources){
1319                     Reference unitReference = is.getCitation();
1320                     String unitMicroReference = is.getCitationMicroReference();
1321
1322                     boolean refMatch=false;
1323                     boolean microMatch=false;
1324
1325                     try{
1326                         if (unitReference==null && reference==null) {
1327                             refMatch=true;
1328                         }
1329                         if (unitReference!=null && reference!=null) {
1330                             if (unitReference.getTitleCache().equalsIgnoreCase(reference.getTitleCache())) {
1331                                 refMatch=true;
1332                             }
1333                         }
1334                     }catch(Exception e){}
1335
1336                     try{
1337                         if (unitMicroReference==null && microReference==null) {
1338                             microMatch=true;
1339                         }
1340                         if(unitMicroReference!=null && microReference!=null) {
1341                             if(unitMicroReference.equalsIgnoreCase(microReference)) {
1342                                 microMatch=true;
1343                             }
1344                         }
1345                     }
1346                     catch(Exception e){}
1347
1348                     if (microMatch && refMatch) {
1349                         return false;
1350                     }
1351                 }
1352                 return true;
1353             }
1354
1355             /**
1356              * look for the Feature object (FieldObs, Specimen,...)
1357              * @param unit : a specimen or obersvation base
1358              * @return the corresponding Feature
1359              */
1360             private Feature makeFeature(SpecimenOrObservationBase<?> unit) {
1361                 SpecimenOrObservationType type = unit.getRecordBasis();
1362
1363
1364
1365                 if (type.isFeatureObservation()){
1366                     return Feature.OBSERVATION();
1367                 }else if (type.isFeatureSpecimen()){
1368                     return Feature.SPECIMEN();
1369                 }else if (type == SpecimenOrObservationType.DerivedUnit){
1370                     return Feature.OBSERVATION();
1371                     //            return getFeature("Specimen or observation");
1372                 }else{
1373                     String message = "Unhandled record basis '%s' for defining individuals association feature type. Use default.";
1374                     logger.warn(String.format(message, type.getMessage()));
1375                     return Feature.OBSERVATION();
1376                     //            return getFeature("Specimen or observation");
1377
1378                 }
1379             }
1380
1381
1382             /**
1383              * @param sourceMap
1384              * @param osbSet
1385              */
1386             protected void addToSourceMap(Map<String, OriginalSourceBase<?>> sourceMap, Set<OriginalSourceBase> osbSet) {
1387                 for( OriginalSourceBase<?> osb:osbSet) {
1388                     if(osb.getCitation()!=null && osb.getCitationMicroReference() !=null  && !osb.getCitationMicroReference().isEmpty()) {
1389                         try{
1390                             sourceMap.put(osb.getCitation().getTitleCache()+ "---"+osb.getCitationMicroReference(),osb);
1391                         }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1392                     } else if(osb.getCitation()!=null){
1393                         try{
1394                             sourceMap.put(osb.getCitation().getTitleCache(),osb);
1395                         }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1396                     }
1397                 }
1398             }
1399
1400 }