cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/SpecimenImportBase.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.specimen;
  11
  12
  13 import java.util.ArrayList;
  14 import java.util.HashMap;
  15 import java.util.HashSet;
  16 import java.util.List;
  17 import java.util.Map;
  18 import java.util.Set;
  19 import java.util.UUID;
  20
  21 import org.apache.log4j.Logger;
  22
  23 import eu.etaxonomy.cdm.api.application.ICdmApplicationConfiguration;
  24 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
  25 import eu.etaxonomy.cdm.api.service.config.FindOccurrencesConfigurator;
  26 import eu.etaxonomy.cdm.api.service.pager.Pager;
  27 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  28 import eu.etaxonomy.cdm.io.common.CdmImportBase;
  29 import eu.etaxonomy.cdm.io.common.IImportConfigurator;
  30 import eu.etaxonomy.cdm.io.specimen.abcd206.in.Identification;
  31 import eu.etaxonomy.cdm.io.specimen.abcd206.in.SpecimenImportReport;
  32 import eu.etaxonomy.cdm.model.agent.AgentBase;
  33 import eu.etaxonomy.cdm.model.agent.Institution;
  34 import eu.etaxonomy.cdm.model.agent.Team;
  35 import eu.etaxonomy.cdm.model.common.CdmBase;
  36 import eu.etaxonomy.cdm.model.common.ISourceable;
  37 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  38 import eu.etaxonomy.cdm.model.common.LanguageString;
  39 import eu.etaxonomy.cdm.model.common.OriginalSourceBase;
  40 import eu.etaxonomy.cdm.model.common.OriginalSourceType;
  41 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  42 import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
  43 import eu.etaxonomy.cdm.model.description.Feature;
  44 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  45 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  46 import eu.etaxonomy.cdm.model.name.BacterialName;
  47 import eu.etaxonomy.cdm.model.name.BotanicalName;
  48 import eu.etaxonomy.cdm.model.name.CultivarPlantName;
  49 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  50 import eu.etaxonomy.cdm.model.name.NonViralName;
  51 import eu.etaxonomy.cdm.model.name.Rank;
  52 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
  53 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
  54 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
  55 import eu.etaxonomy.cdm.model.name.ZoologicalName;
  56 import eu.etaxonomy.cdm.model.occurrence.Collection;
  57 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  58 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
  59 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
  60 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationType;
  61 import eu.etaxonomy.cdm.model.reference.Reference;
  62 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  63 import eu.etaxonomy.cdm.model.taxon.Classification;
  64 import eu.etaxonomy.cdm.model.taxon.Synonym;
  65 import eu.etaxonomy.cdm.model.taxon.Taxon;
  66 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  67 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  68 import eu.etaxonomy.cdm.persistence.dto.UuidAndTitleCache;
  69 import eu.etaxonomy.cdm.persistence.query.MatchMode;
  70 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
  71 import eu.etaxonomy.cdm.strategy.parser.ParserProblem;
  72
  73
  74 /**
  75  * @author p.kelbert
  76  * @created 20.10.2008
  77  */
  78 public abstract class SpecimenImportBase<CONFIG extends IImportConfigurator, STATE extends SpecimenImportStateBase>  extends CdmImportBase<CONFIG, STATE> {
  79
  80     private static final long serialVersionUID = 4423065367998125678L;
  81     private static final Logger logger = Logger.getLogger(SpecimenImportBase.class);
  82         protected final boolean DEBUG = true;
  83
  84         protected static final UUID SPECIMEN_SCAN_TERM = UUID.fromString("acda15be-c0e2-4ea8-8783-b9b0c4ad7f03");
  85
  86         private static final String COLON = ":";
  87
  88
  89         @Override
  90     protected abstract void doInvoke(STATE state);
  91
  92         /**
  93          * Handle a single unit
  94          * @param state
  95          * @param item
  96          */
  97         protected abstract void handleSingleUnit(STATE state, Object item) ;
  98
  99
 100
 101         protected TaxonNameBase<?, ?> getOrCreateTaxonName(String scientificName, Rank rank, boolean preferredFlag, STATE state, int unitIndexInAbcdFile){
 102         TaxonNameBase<?, ?> taxonName = null;
 103         SpecimenImportConfiguratorBase config = state.getConfig();
 104
 105         //check atomised name data for rank
 106         //new name will be created
 107         NonViralName<?> atomisedTaxonName = null;
 108         if (rank==null && unitIndexInAbcdFile>=0 && ((state.getDataHolder().getAtomisedIdentificationList() != null && !state.getDataHolder().getAtomisedIdentificationList().isEmpty())|| state.getDataHolder().getAtomisedIdentificationList().size() > 0)) {
 109             atomisedTaxonName = setTaxonNameByType(state.getDataHolder().getAtomisedIdentificationList().get(unitIndexInAbcdFile), scientificName, state);
 110             if(atomisedTaxonName!=null){
 111                 rank = atomisedTaxonName.getRank();
 112             }
 113         }
 114         if(config.isReuseExistingTaxaWhenPossible()){
 115             NonViralName<?> parsedName = atomisedTaxonName;
 116             if(parsedName==null){
 117                 parsedName = parseScientificName(scientificName, state, state.getReport());
 118             }
 119             atomisedTaxonName = parsedName;
 120             if(config.isIgnoreAuthorship() && parsedName!=null && preferredFlag){
 121                 // do not ignore authorship for non-preferred names because they need
 122                 // to be created for the determination history
 123                 String nameCache = parsedName.getNameCache();
 124                 List<NonViralName> names = getNameService().findNamesByNameCache(nameCache, MatchMode.EXACT, null);
 125                 if (!names.isEmpty()){
 126                      return getBestMatchingName(scientificName, new ArrayList<TaxonNameBase>(names), state);
 127                 }
 128             } else {
 129                 //search for existing names
 130                 List<TaxonNameBase> names = getNameService().listByTitle(TaxonNameBase.class, scientificName, MatchMode.EXACT, null, null, null, null, null);
 131                 taxonName = getBestMatchingName(scientificName, names, state);
 132                 //still nothing found -> try with the atomised name full title cache
 133                 if(taxonName==null && atomisedTaxonName!=null){
 134                     names = getNameService().listByTitle(TaxonNameBase.class, atomisedTaxonName.getFullTitleCache(), MatchMode.EXACT, null, null, null, null, null);
 135                     taxonName = getBestMatchingName(atomisedTaxonName.getTitleCache(), names, state);
 136                     //still nothing found -> try with the atomised name title cache
 137                     if(taxonName==null){
 138                         names = getNameService().listByTitle(TaxonNameBase.class, atomisedTaxonName.getTitleCache(), MatchMode.EXACT, null, null, null, null, null);
 139                         taxonName = getBestMatchingName(atomisedTaxonName.getTitleCache(), names, state);
 140                     }
 141                 }
 142
 143             }
 144
 145         }
 146
 147         if(taxonName==null && atomisedTaxonName!=null){
 148             taxonName = atomisedTaxonName;
 149             state.getReport().addName(taxonName);
 150             logger.info("Created new taxon name "+taxonName);
 151             if(taxonName.hasProblem()){
 152                 state.getReport().addInfoMessage(String.format("Created %s with parsing problems", taxonName));
 153             }
 154             if(!atomisedTaxonName.getTitleCache().equals(scientificName)){
 155                 state.getReport().addInfoMessage(String.format("Taxon %s was parsed as %s", scientificName, atomisedTaxonName.getTitleCache()));
 156             }
 157         }
 158         else if(taxonName==null){
 159             //create new taxon name
 160
 161             if (state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNAFP)){
 162                 taxonName = BotanicalName.NewInstance(rank);
 163             }else if (state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICZN)){
 164                 taxonName = ZoologicalName.NewInstance(rank);
 165             }else{
 166                 taxonName = NonViralName.NewInstance(rank);
 167             }
 168             taxonName.setFullTitleCache(scientificName,true);
 169             taxonName.setTitleCache(scientificName, true);
 170             state.getReport().addName(taxonName);
 171             logger.info("Created new taxon name "+taxonName);
 172         }
 173         save(taxonName, state);
 174         return taxonName;
 175     }
 176
 177          protected TaxonNameBase<?, ?> getBestMatchingName(String scientificName, java.util.Collection<TaxonNameBase> names, STATE state){
 178                 List<TaxonNameBase> namesWithAcceptedTaxa = new ArrayList<TaxonNameBase>();
 179                 for (TaxonNameBase name : names) {
 180                     if(!name.getTaxa().isEmpty()){
 181                         namesWithAcceptedTaxa.add(name);
 182                     }
 183                 }
 184                 String message = "More than one taxon name was found for "+scientificName+"!";
 185                 //check for names with accepted taxa
 186                 if(namesWithAcceptedTaxa.size()>0){
 187                     if(namesWithAcceptedTaxa.size()>1){
 188                         state.getReport().addInfoMessage(message);
 189                         logger.warn(message);
 190                         return null;
 191                     }
 192                     return namesWithAcceptedTaxa.iterator().next();
 193                 }
 194                 //no names with accepted taxa found -> check accepted taxa of synonyms
 195                 List<Taxon> taxaFromSynonyms = new ArrayList<Taxon>();
 196                 for (TaxonNameBase name : names) {
 197                     Set<TaxonBase> taxonBases = name.getTaxonBases();
 198                     for (TaxonBase taxonBase : taxonBases) {
 199                         if(taxonBase.isInstanceOf(Synonym.class)){
 200                             Synonym synonym = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
 201                             taxaFromSynonyms.add(synonym.getAcceptedTaxon());
 202                         }
 203                     }
 204                 }
 205                 if(taxaFromSynonyms.size()>0){
 206                     if(taxaFromSynonyms.size()>1){
 207                         state.getReport().addInfoMessage(message);
 208                         logger.warn(message);
 209                         return null;
 210                     }
 211                     return taxaFromSynonyms.iterator().next().getName();
 212                 }
 213                 return null;
 214             }
 215          /**
 216              * Parse automatically the scientific name
 217              * @param scientificName the scientific name to parse
 218              * @param state the current import state
 219              * @param report the import report
 220              * @return a parsed name
 221              */
 222             protected NonViralName<?> parseScientificName(String scientificName, STATE state, SpecimenImportReport report) {
 223                 NonViralNameParserImpl nvnpi = NonViralNameParserImpl.NewInstance();
 224                 NonViralName<?> taxonName = null;
 225                 boolean problem = false;
 226
 227                 if(DEBUG){
 228                     logger.info("parseScientificName " + state.getDataHolder().getNomenclatureCode().toString());
 229                 }
 230
 231                 if (state.getDataHolder().getNomenclatureCode().toString().equals("Zoological") || state.getDataHolder().getNomenclatureCode().toString().contains("ICZN")) {
 232                     taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICZN, null);
 233                     if (taxonName.hasProblem()) {
 234                         problem = true;
 235                     }
 236                 }
 237                 else if (state.getDataHolder().getNomenclatureCode().toString().equals("Botanical") || state.getDataHolder().getNomenclatureCode().toString().contains("ICBN")) {
 238                     taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNAFP, null);
 239                     if (taxonName.hasProblem()) {
 240                         problem = true;
 241                     }
 242                 }
 243                 else if (state.getDataHolder().getNomenclatureCode().toString().equals("Bacterial") || state.getDataHolder().getNomenclatureCode().toString().contains("ICBN")) {
 244                     taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNB, null);
 245                     if (taxonName.hasProblem()) {
 246                         problem = true;
 247                     }
 248                 }
 249                 else if (state.getDataHolder().getNomenclatureCode().toString().equals("Cultivar") || state.getDataHolder().getNomenclatureCode().toString().contains("ICNCP")) {
 250                     taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNCP, null);
 251                     if (taxonName.hasProblem()) {
 252                         problem = true;
 253                     }
 254                 }
 255                 if (problem) {
 256                     String message = String.format("Parsing problems for %s", scientificName);
 257                     if(taxonName!=null){
 258                         for (ParserProblem parserProblem : taxonName.getParsingProblems()) {
 259                             message += "\n\t- "+parserProblem;
 260                         }
 261                     }
 262                     report.addInfoMessage(message);
 263                     logger.info(message);
 264                 }
 265                 return taxonName;
 266
 267             }
 268
 269             /**
 270              * Create the name without automatic parsing, either because it failed, or because the user deactivated it.
 271              * The name is built upon the ABCD fields
 272              * @param atomisedMap : the ABCD atomised fields
 273              * @param fullName : the full scientific name
 274              * @param state
 275              * @return the corresponding Botanical or Zoological or... name
 276              */
 277             protected NonViralName<?> setTaxonNameByType(
 278                     HashMap<String, String> atomisedMap, String fullName, STATE state) {
 279                 boolean problem = false;
 280                 if(DEBUG) {
 281                     logger.info("settaxonnamebytype " + state.getDataHolder().getNomenclatureCode().toString());
 282                 }
 283
 284                 if (state.getDataHolder().getNomenclatureCode().equals("Zoological") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICZN.getUuid())) {
 285                     NonViralName<ZoologicalName> taxonName = ZoologicalName.NewInstance(null);
 286                     taxonName.setFullTitleCache(fullName, true);
 287                     taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
 288                     taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
 289                     taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap,"SpeciesEpithet")));
 290                     taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap,"SubspeciesEpithet")));
 291
 292                     if (taxonName.getGenusOrUninomial() != null){
 293                         taxonName.setRank(Rank.GENUS());
 294                     }
 295
 296                     if (taxonName.getInfraGenericEpithet() != null){
 297                         taxonName.setRank(Rank.SUBGENUS());
 298                     }
 299
 300                     if (taxonName.getSpecificEpithet() != null){
 301                         taxonName.setRank(Rank.SPECIES());
 302                     }
 303
 304                     if (taxonName.getInfraSpecificEpithet() != null){
 305                         taxonName.setRank(Rank.SUBSPECIES());
 306                     }
 307
 308                     Team team = null;
 309                     if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 310                         team = Team.NewInstance();
 311                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
 312                     }
 313                     else {
 314                         if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 315                             team = Team.NewInstance();
 316                             team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
 317                         }
 318                     }
 319                     if (team != null) {
 320                         taxonName.setBasionymAuthorship(team);
 321                     }
 322                     else {
 323                         if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 324                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
 325                         }
 326                         else if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 327                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamAndYear"));
 328                         }
 329                     }
 330                     if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
 331                         team = Team.NewInstance();
 332                         team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
 333                         taxonName.setCombinationAuthorship(team);
 334                     }
 335                     if (taxonName.hasProblem()) {
 336                         logger.info("pb ICZN");
 337                         problem = true;
 338                     }
 339                     else {
 340                         return taxonName;
 341                     }
 342                 }
 343                 else if (state.getDataHolder().getNomenclatureCode().equals("Botanical") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNAFP.getUuid())) {
 344                     BotanicalName taxonName = (BotanicalName) parseScientificName(fullName, state, state.getReport());
 345                     if (taxonName != null){
 346                         return taxonName;
 347                     }
 348                     else{
 349                         taxonName = BotanicalName.NewInstance(null);
 350                     }
 351                     taxonName.setFullTitleCache(fullName, true);
 352                     taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
 353                     taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "FirstEpithet")));
 354                     taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "InfraSpeEpithet")));
 355                     try {
 356                         taxonName.setRank(Rank.getRankByName(getFromMap(atomisedMap, "Rank")));
 357                     } catch (Exception e) {
 358                         if (taxonName.getInfraSpecificEpithet() != null){
 359                             taxonName.setRank(Rank.SUBSPECIES());
 360                         }
 361                         else if (taxonName.getSpecificEpithet() != null){
 362                             taxonName.setRank(Rank.SPECIES());
 363                         }
 364                         else if (taxonName.getInfraGenericEpithet() != null){
 365                             taxonName.setRank(Rank.SUBGENUS());
 366                         }
 367                         else if (taxonName.getGenusOrUninomial() != null){
 368                             taxonName.setRank(Rank.GENUS());
 369                         }
 370                     }
 371                     Team team = null;
 372                     if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 373                         team = Team.NewInstance();
 374                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
 375                         taxonName.setBasionymAuthorship(team);
 376                     }
 377                     if (getFromMap(atomisedMap, "AuthorTeam") != null) {
 378                         team = Team.NewInstance();
 379                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeam"), true);
 380                         taxonName.setCombinationAuthorship(team);
 381                     }
 382                     if (team == null) {
 383                         if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 384                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
 385                         }
 386                         else if (getFromMap(atomisedMap, "AuthorTeam") != null) {
 387                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeam"));
 388                         }
 389                     }
 390                     if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
 391                         team = Team.NewInstance();
 392                         team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
 393                         taxonName.setCombinationAuthorship(team);
 394                     }
 395                     if (taxonName.hasProblem()) {
 396                         logger.info("pb ICBN");
 397                         problem = true;
 398                     }
 399                     else {
 400                         return taxonName;
 401                     }
 402                 }
 403                 else if (state.getDataHolder().getNomenclatureCode().equals("Bacterial") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNB.getUuid())) {
 404                     NonViralName<BacterialName> taxonName = BacterialName.NewInstance(null);
 405                     taxonName.setFullTitleCache(fullName, true);
 406                     taxonName.setGenusOrUninomial(getFromMap(atomisedMap, "Genus"));
 407                     taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
 408                     taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "Species")));
 409                     taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "SubspeciesEpithet")));
 410
 411                     if (taxonName.getGenusOrUninomial() != null){
 412                         taxonName.setRank(Rank.GENUS());
 413                     }
 414                     else if (taxonName.getInfraGenericEpithet() != null){
 415                         taxonName.setRank(Rank.SUBGENUS());
 416                     }
 417                     else if (taxonName.getSpecificEpithet() != null){
 418                         taxonName.setRank(Rank.SPECIES());
 419                     }
 420                     else if (taxonName.getInfraSpecificEpithet() != null){
 421                         taxonName.setRank(Rank.SUBSPECIES());
 422                     }
 423
 424                     if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 425                         Team team = Team.NewInstance();
 426                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
 427                         taxonName.setCombinationAuthorship(team);
 428                     }
 429                     if (getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear") != null) {
 430                         Team team = Team.NewInstance();
 431                         team.setTitleCache(getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear"), true);
 432                         taxonName.setBasionymAuthorship(team);
 433                     }
 434                     if (taxonName.hasProblem()) {
 435                         logger.info("pb ICNB");
 436                         problem = true;
 437                     }
 438                     else {
 439                         return taxonName;
 440                     }
 441                 }
 442                 else if (state.getDataHolder().getNomenclatureCode().equals("Cultivar")) {
 443                     CultivarPlantName taxonName = CultivarPlantName.NewInstance(null);
 444
 445                     if (taxonName.hasProblem()) {
 446                         logger.info("pb ICNCP");
 447                         problem = true;
 448                     }
 449                     else {
 450                         return taxonName;
 451                     }
 452                     return taxonName;
 453                 }
 454
 455                 if (problem) {
 456                     logger.info("Problem im setTaxonNameByType ");
 457                     NonViralName<?> taxonName = NonViralName.NewInstance(null);
 458                     taxonName.setFullTitleCache(fullName, true);
 459                     return taxonName;
 460                 }
 461                 NonViralName<?> tn = NonViralName.NewInstance(null);
 462                 return tn;
 463             }
 464
 465             /**
 466              * Get a formated string from a hashmap
 467              * @param atomisedMap
 468              * @param key
 469              * @return
 470              */
 471             private String getFromMap(HashMap<String, String> atomisedMap, String key) {
 472                 String value = null;
 473                 if (atomisedMap.containsKey(key)) {
 474                     value = atomisedMap.get(key);
 475                 }
 476
 477                 try {
 478                     if (value != null && key.matches(".*Year.*")) {
 479                         value = value.trim();
 480                         if (value.matches("[a-z A-Z ]*[0-9]{4}$")) {
 481                             String tmp = value.split("[0-9]{4}$")[0];
 482                             int year = Integer.parseInt(value.split(tmp)[1]);
 483                             if (year >= 1752) {
 484                                 value = tmp;
 485                             }
 486                             else {
 487                                 value = null;
 488                             }
 489                         }
 490                         else {
 491                             value = null;
 492                         }
 493                     }
 494                 }
 495                 catch (Exception e) {
 496                     value = null;
 497                 }
 498                 return value;
 499             }
 500
 501             /**
 502              * Very fast and dirty implementation to allow handling of transient objects as described in
 503              * https://dev.e-taxonomy.eu/trac/ticket/3726
 504              *
 505              * Not yet complete.
 506              *
 507              * @param cdmBase
 508              * @param state
 509              */
 510             protected UUID save(CdmBase cdmBase, SpecimenImportStateBase state) {
 511                 ICdmApplicationConfiguration cdmRepository = state.getConfig().getCdmAppController();
 512                 if (cdmRepository == null){
 513                     cdmRepository = this;
 514                 }
 515
 516                 if (cdmBase.isInstanceOf(LanguageString.class)){
 517                     return cdmRepository.getTermService().saveLanguageData(CdmBase.deproxy(cdmBase, LanguageString.class));
 518                 }else if (cdmBase.isInstanceOf(SpecimenOrObservationBase.class)){
 519                     return cdmRepository.getOccurrenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, SpecimenOrObservationBase.class));
 520                 }else if (cdmBase.isInstanceOf(Reference.class)){
 521                     return cdmRepository.getReferenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, Reference.class));
 522                 }else if (cdmBase.isInstanceOf(Classification.class)){
 523                     return cdmRepository.getClassificationService().saveOrUpdate(CdmBase.deproxy(cdmBase, Classification.class));
 524                 }else if (cdmBase.isInstanceOf(AgentBase.class)){
 525                     return cdmRepository.getAgentService().saveOrUpdate(CdmBase.deproxy(cdmBase, AgentBase.class));
 526                 }else if (cdmBase.isInstanceOf(Collection.class)){
 527                     return cdmRepository.getCollectionService().saveOrUpdate(CdmBase.deproxy(cdmBase, Collection.class));
 528                 }else if (cdmBase.isInstanceOf(DescriptionBase.class)){
 529                     return cdmRepository.getDescriptionService().saveOrUpdate(CdmBase.deproxy(cdmBase, DescriptionBase.class));
 530                 }else if (cdmBase.isInstanceOf(TaxonBase.class)){
 531                     return cdmRepository.getTaxonService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonBase.class));
 532                 }else if (cdmBase.isInstanceOf(TaxonNameBase.class)){
 533                     return cdmRepository.getNameService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonNameBase.class));
 534                 }else if (cdmBase.isInstanceOf(TaxonNode.class)){
 535                 return cdmRepository.getTaxonNodeService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonNode.class));
 536             }else{
 537                     throw new IllegalArgumentException("Class not supported in save method: " + CdmBase.deproxy(cdmBase, CdmBase.class).getClass().getSimpleName());
 538                 }
 539
 540             }
 541
 542
 543             protected SpecimenOrObservationBase findExistingSpecimen(String unitId, SpecimenImportStateBase state){
 544                 ICdmApplicationConfiguration cdmAppController = state.getConfig().getCdmAppController();
 545                 if(cdmAppController==null){
 546                     cdmAppController = this;
 547                 }
 548                 FindOccurrencesConfigurator config = new FindOccurrencesConfigurator();
 549                 config.setSignificantIdentifier(unitId);
 550                 Pager<SpecimenOrObservationBase> existingSpecimens = cdmAppController.getOccurrenceService().findByTitle(config);
 551                 if(!existingSpecimens.getRecords().isEmpty()){
 552                     if(existingSpecimens.getRecords().size()==1){
 553                         return existingSpecimens.getRecords().iterator().next();
 554                     }
 555                 }
 556                 return null;
 557             }
 558
 559             protected abstract void importAssociatedUnits(STATE state, Object item, DerivedUnitFacade derivedUnitFacade);
 560
 561             /**
 562              * getFacade : get the DerivedUnitFacade based on the recordBasis
 563              * @param state
 564              *
 565              * @return DerivedUnitFacade
 566              */
 567             protected DerivedUnitFacade getFacade(STATE state) {
 568                 if(DEBUG) {
 569                     logger.info("getFacade()");
 570                 }
 571                 SpecimenOrObservationType type = null;
 572
 573                 // create specimen
 574                 if (NB((state.getDataHolder().getRecordBasis())) != null) {
 575                     if (state.getDataHolder().getRecordBasis().toLowerCase().startsWith("s") || state.getDataHolder().getRecordBasis().toLowerCase().indexOf("specimen")>-1) {// specimen
 576                         type = SpecimenOrObservationType.PreservedSpecimen;
 577                     }
 578                     if (state.getDataHolder().getRecordBasis().toLowerCase().startsWith("o") ||state.getDataHolder().getRecordBasis().toLowerCase().indexOf("observation")>-1 ) {
 579                         type = SpecimenOrObservationType.Observation;
 580                     }
 581                     if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("fossil")>-1){
 582                         type = SpecimenOrObservationType.Fossil;
 583                     }
 584                     if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("living")>-1) {
 585                         type = SpecimenOrObservationType.LivingSpecimen;
 586                     }
 587                     if (type == null) {
 588                         logger.info("The basis of record does not seem to be known: " + state.getDataHolder().getRecordBasis());
 589                         type = SpecimenOrObservationType.DerivedUnit;
 590                     }
 591                     // TODO fossils?
 592                 } else {
 593                     logger.info("The basis of record is null");
 594                     type = SpecimenOrObservationType.DerivedUnit;
 595                 }
 596                 DerivedUnitFacade derivedUnitFacade = DerivedUnitFacade.NewInstance(type);
 597                 return derivedUnitFacade;
 598             }
 599
 600             /**
 601              * Look if the Institution does already exist
 602              * @param institutionCode: a string with the institutioncode
 603              * @param config : the configurator
 604              * @return the Institution (existing or new)
 605              */
 606             protected Institution getInstitution(String institutionCode, STATE state) {
 607                 SpecimenImportConfiguratorBase config = state.getConfig();
 608                 Institution institution=null;
 609                 institution = (Institution)state.institutions.get(institutionCode);
 610                 if (institution != null){
 611                     return institution;
 612                 }
 613                 List<Institution> institutions;
 614                 try {
 615                     institutions = getAgentService().searchInstitutionByCode(institutionCode);
 616
 617                 } catch (Exception e) {
 618                     institutions = new ArrayList<Institution>();
 619                     logger.warn(e);
 620                 }
 621                 if (institutions.size() > 0 && config.isReuseExistingMetaData()) {
 622                     for (Institution institut:institutions){
 623                         try{
 624                             if (institut.getCode().equalsIgnoreCase(institutionCode)) {
 625                                 institution=institut;
 626                                 break;
 627                             }
 628                         }catch(Exception e){logger.warn("no institution code in the db");}
 629                     }
 630                 }
 631                 if(DEBUG) {
 632                     if(institution !=null) {
 633                         logger.info("getinstitution " + institution.toString());
 634                     }
 635                 }
 636                 if (institution == null){
 637                     // create institution
 638                     institution = Institution.NewInstance();
 639                     institution.setCode(institutionCode);
 640                     institution.setTitleCache(institutionCode, true);
 641                     UUID uuid = save(institution, state);
 642                 }
 643
 644
 645                 state.institutions.put(institutionCode, institution);
 646                 return institution;
 647             }
 648
 649             /**
 650              * Look if the Collection does already exist
 651              * @param collectionCode
 652              * @param collectionCode: a string
 653              * @param config : the configurator
 654              * @return the Collection (existing or new)
 655              */
 656             protected Collection getCollection(Institution institution, String collectionCode, STATE state) {
 657                 SpecimenImportConfiguratorBase config = state.getConfig();
 658                 Collection collection = null;
 659                 List<Collection> collections;
 660                 collection = (Collection) state.collections.get(collectionCode);
 661                 if (collection != null){
 662                     return collection;
 663                 }
 664                 try {
 665                     collections = getCollectionService().searchByCode(collectionCode);
 666                 } catch (Exception e) {
 667                     collections = new ArrayList<Collection>();
 668                 }
 669                 if (collections.size() > 0 && config.isReuseExistingMetaData()) {
 670                     for (Collection coll:collections){
 671                         if (coll.getCode() != null && coll.getInstitute() != null
 672                                 && coll.getCode().equalsIgnoreCase(collectionCode) && coll.getInstitute().equals(institution)) {
 673                             collection = coll;
 674                             break;
 675                         }
 676                     }
 677                 }
 678
 679                 if(collection == null){
 680                     collection =Collection.NewInstance();
 681                     collection.setCode(collectionCode);
 682                     collection.setInstitute(institution);
 683                     collection.setTitleCache(collectionCode);
 684                     UUID uuid = save(collection, state);
 685                 }
 686
 687
 688
 689                 state.collections.put(collectionCode, collection);
 690
 691                 return collection;
 692             }
 693
 694             /**
 695              * @param reference
 696              * @param citationDetail
 697              * @return
 698              */
 699             //FIXME this method is highly critical, because
 700             //  * it will have serious performance and memory problems with large databases
 701             //        (databases may easily have >1 Mio source records)
 702             //  * it does not make sense to search for existing sources and then clone them
 703             //    we need to search for existing references instead and use them (if exist)
 704             //    for our new source.
 705             protected IdentifiableSource getIdentifiableSource(Reference reference, String citationDetail) {
 706
 707               /*  List<IdentifiableSource> issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
 708
 709
 710                 if (reference != null){
 711                     try {
 712                         for (OriginalSourceBase<?> osb: issTmp){
 713                             if (osb.getCitation() != null && osb.getCitation().getTitleCache().equalsIgnoreCase(reference.getTitleCache())){
 714                                 String osbDetail = osb.getCitationMicroReference();
 715                                 if ((StringUtils.isBlank(osbDetail) && StringUtils.isBlank(citationDetail))
 716                                         || (osbDetail != null && osbDetail.equalsIgnoreCase(citationDetail)) ) {
 717 //                                  System.out.println("REFERENCE FOUND RETURN EXISTING SOURCE");
 718                                     return (IdentifiableSource) osb.clone();
 719                                 }
 720                             }
 721                         }
 722                     } catch (CloneNotSupportedException e) {
 723                         throw new RuntimeException(e);
 724                     } catch (Exception e1){
 725                         e1.printStackTrace();
 726                     }
 727                 }
 728         */
 729                 IdentifiableSource sour = IdentifiableSource.NewInstance(OriginalSourceType.Import,null,null, reference,citationDetail);
 730                 return sour;
 731             }
 732
 733             /**
 734              * Add the hierarchy for a Taxon(add higher taxa)
 735              * @param classification
 736              * @param taxon: a taxon to add as a node
 737              * @param state: the ABCD import state
 738              */
 739             protected void addParentTaxon(Taxon taxon, STATE state, boolean preferredFlag, Classification classification){
 740                 NonViralName<?>  nvname = CdmBase.deproxy(taxon.getName(), NonViralName.class);
 741                 Rank rank = nvname.getRank();
 742                 Taxon genus =null;
 743                 Taxon subgenus =null;
 744                 Taxon species = null;
 745                 Taxon subspecies = null;
 746                 Taxon parent = null;
 747                 if(rank!=null){
 748                     if (rank.isLower(Rank.GENUS() )){
 749                         String genusOrUninomial = nvname.getGenusOrUninomial();
 750                         TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(genusOrUninomial, Rank.GENUS(), preferredFlag, state, -1);
 751                         genus = getOrCreateTaxonForName(taxonName, state);
 752                         if (preferredFlag) {
 753                             parent = linkParentChildNode(null, genus, classification, state);
 754                         }
 755
 756                     }
 757                     if (rank.isLower(Rank.SUBGENUS())){
 758                         String prefix = nvname.getGenusOrUninomial();
 759                         String name = nvname.getInfraGenericEpithet();
 760                         if (name != null){
 761                             TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(prefix+" "+name, Rank.SUBGENUS(), preferredFlag, state, -1);
 762                             subgenus = getOrCreateTaxonForName(taxonName, state);
 763                             if (preferredFlag) {
 764                                 parent = linkParentChildNode(genus, subgenus, classification, state);
 765                             }            }
 766                     }
 767                     if (rank.isLower(Rank.SPECIES())){
 768                         if (subgenus!=null){
 769                             String prefix = nvname.getGenusOrUninomial();
 770                             String name = nvname.getInfraGenericEpithet();
 771                             String spe = nvname.getSpecificEpithet();
 772                             if (spe != null){
 773                                 TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(prefix+" "+name+" "+spe, Rank.SPECIES(), preferredFlag, state, -1);
 774                                 species = getOrCreateTaxonForName(taxonName, state);
 775                                 if (preferredFlag) {
 776                                     parent = linkParentChildNode(subgenus, species, classification, state);
 777                                 }
 778                             }
 779                         }
 780                         else{
 781                             String prefix = nvname.getGenusOrUninomial();
 782                             String name = nvname.getSpecificEpithet();
 783                             if (name != null){
 784                                 TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(prefix+" "+name, Rank.SPECIES(), preferredFlag, state, -1);
 785                                 species = getOrCreateTaxonForName(taxonName, state);
 786                                 if (preferredFlag) {
 787                                     parent = linkParentChildNode(genus, species, classification, state);
 788                                 }
 789                             }
 790                         }
 791                     }
 792                     if (rank.isLower(Rank.INFRASPECIES())){
 793                         TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(nvname.getFullTitleCache(), Rank.SUBSPECIES(), preferredFlag, state, -1);
 794                         subspecies = getOrCreateTaxonForName(taxonName, state);
 795                         if (preferredFlag) {
 796                             parent = linkParentChildNode(species, subspecies, classification, state);
 797                         }
 798                     }
 799                 }
 800                 if (preferredFlag && parent!=taxon) {
 801                     linkParentChildNode(parent, taxon, classification, state);
 802                 }
 803             }
 804
 805             /**
 806              * Link a parent to a child and save it in the current classification
 807              * @param parent: the higher Taxon
 808              * @param child : the lower (or current) Taxon
 809              * return the Taxon from the new created Node
 810              * @param classification
 811              * @param state
 812              */
 813             protected Taxon linkParentChildNode(Taxon parent, Taxon child, Classification classification, STATE state) {
 814                 TaxonNode node =null;
 815                 if (parent != null) {
 816                     parent = (Taxon) getTaxonService().find(parent.getUuid());
 817                     child = (Taxon) getTaxonService().find(child.getUuid());
 818                     //here we do not have to check if the taxon nodes already exists
 819                     //this is done by classification.addParentChild()
 820                     //do not add child node if it already exists
 821                     if(hasTaxonNodeInClassification(child, classification)){
 822                         return child;
 823                     }
 824                     else{
 825                         node = classification.addParentChild(parent, child, state.getRef(), "");
 826                         save(node, state);
 827                     }
 828                 }
 829                 else {
 830                     child = (Taxon) getTaxonService().find(child.getUuid());
 831                     //do not add child node if it already exists
 832                     if(hasTaxonNodeInClassification(child, classification)){
 833                         return child;
 834                     }
 835                     else{
 836                         node = classification.addChildTaxon(child, state.getRef(), null);
 837                         save(node, state);
 838                     }
 839                 }
 840                 if(node!=null){
 841                     state.getReport().addTaxonNode(node);
 842                     return node.getTaxon();
 843                 }
 844                 String message = "Could not create taxon node for " +child;
 845                 state.getReport().addInfoMessage(message);
 846                 logger.warn(message);
 847                 return null;
 848             }
 849
 850             protected Taxon getOrCreateTaxonForName(TaxonNameBase<?, ?> taxonNameBase, STATE state){
 851
 852                 Set<Taxon> acceptedTaxa = taxonNameBase.getTaxa();
 853                 if(acceptedTaxa.size()>0){
 854                     Taxon firstAcceptedTaxon = acceptedTaxa.iterator().next();
 855                     if(acceptedTaxa.size()>1){
 856                         String message = "More than one accepted taxon was found for taxon name: "
 857                                 + taxonNameBase.getTitleCache() + "!\n" + firstAcceptedTaxon + "was chosen for "+state.getDerivedUnitBase();
 858                         state.getReport().addInfoMessage(message);
 859                         logger.warn(message);
 860                     }
 861                     else{
 862                         return firstAcceptedTaxon;
 863                     }
 864                 }
 865                 else{
 866                     Set<TaxonBase> taxonAndSynonyms = taxonNameBase.getTaxonBases();
 867                     for (TaxonBase taxonBase : taxonAndSynonyms) {
 868                         if(taxonBase.isInstanceOf(Synonym.class)){
 869                             Synonym synonym = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
 870                             Taxon acceptedTaxonOfSynonym = synonym.getAcceptedTaxon();
 871                             if(acceptedTaxonOfSynonym == null){
 872                                 String message = "No accepted taxon could be found for taxon name: "
 873                                         + taxonNameBase.getTitleCache()
 874                                         + "!";
 875                                 state.getReport().addInfoMessage(message);
 876                                 logger.warn(message);
 877                             }
 878                             else{
 879                                 return acceptedTaxonOfSynonym;
 880                             }
 881                         }
 882                     }
 883                 }
 884                 Taxon taxon = Taxon.NewInstance(taxonNameBase, state.getRef());
 885                 save(taxon, state);
 886                 state.getReport().addTaxon(taxon);
 887                 logger.info("Created new taxon "+ taxon);
 888                 return taxon;
 889             }
 890
 891             private boolean hasTaxonNodeInClassification(Taxon taxon, Classification classification){
 892                 if(taxon.getTaxonNodes()!=null){
 893                     for (TaxonNode node : taxon.getTaxonNodes()){
 894                         if(node.getClassification().equals(classification)){
 895                             return true;
 896                         }
 897                     }
 898                 }
 899                 return false;
 900             }
 901
 902             /**
 903              * HandleIdentifications : get the scientific names present in the ABCD
 904              * document and store link them with the observation/specimen data
 905              * @param state: the current ABCD import state
 906              * @param derivedUnitFacade : the current derivedunitfacade
 907              */
 908             protected void handleIdentifications(STATE state, DerivedUnitFacade derivedUnitFacade) {
 909                SpecimenImportConfiguratorBase config = state.getConfig();
 910
 911
 912                 String scientificName = "";
 913                 boolean preferredFlag = false;
 914
 915                 if (state.getDataHolder().getNomenclatureCode() == ""){
 916                     if (config.getNomenclaturalCode() != null){
 917                         if (config.getNomenclaturalCode() != null){
 918                             state.getDataHolder().setNomenclatureCode(config.getNomenclaturalCode().toString());
 919                         }
 920                     }
 921                 }
 922
 923                 for (int i = 0; i < state.getDataHolder().getIdentificationList().size(); i++) {
 924                     Identification identification = state.getDataHolder().getIdentificationList().get(i);
 925                     scientificName = identification.getScientificName().replaceAll(" et ", " & ");
 926
 927                     String preferred = identification.getPreferred();
 928                     if (preferred.equals("1") || preferred.toLowerCase().indexOf("true") != -1 || state.getDataHolder().getIdentificationList().size()==1) {
 929                         preferredFlag = true;
 930                     }
 931                     else {
 932                         preferredFlag = false;
 933                     }
 934                     if (identification.getCode() != null){
 935                     if (identification.getCode().indexOf(':') != -1) {
 936                         state.getDataHolder().setNomenclatureCode(identification.getCode().split(COLON)[1]);
 937                     }
 938                     else{
 939                         state.getDataHolder().setNomenclatureCode(identification.getCode());
 940                     }
 941                     }
 942                     TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(scientificName, null, preferredFlag, state, i);
 943                     Taxon taxon = getOrCreateTaxonForName(taxonName, state);
 944                     addTaxonNode(taxon, state,preferredFlag);
 945                     linkDeterminationEvent(state, taxon, preferredFlag, derivedUnitFacade);
 946                 }
 947             }
 948
 949             /**
 950              * @param taxon : a taxon to add as a node
 951              * @param state : the ABCD import state
 952              */
 953             protected void addTaxonNode(Taxon taxon, STATE state, boolean preferredFlag) {
 954                 SpecimenImportConfiguratorBase config = state.getConfig();
 955                 logger.info("link taxon to a taxonNode "+taxon.getTitleCache());
 956                 //only add nodes if not already existing in current classification or default classification
 957
 958                 //check if node exists in current classification
 959                 //NOTE: we cannot use hasTaxonNodeInClassification() here because we are first creating it here
 960                 if (!existsInClassification(taxon,state.getClassification(), state)){
 961                     if(config.isMoveNewTaxaToDefaultClassification()){
 962                         //check if node exists in default classification
 963                         if (!existsInClassification(taxon, state.getDefaultClassification(), state)){
 964                             addParentTaxon(taxon, state, preferredFlag, state.getDefaultClassification());
 965                         }
 966                     }
 967                     else {
 968                         //add non-existing taxon to current classification
 969                         addParentTaxon(taxon, state, preferredFlag, state.getClassification());
 970                     }
 971                 }
 972             }
 973
 974
 975             private boolean existsInClassification(Taxon taxon, Classification classification, STATE state){
 976                 boolean exist = false;
 977                 ICdmApplicationConfiguration cdmAppController = state.getConfig().getCdmAppController();
 978             if(cdmAppController==null){
 979                 cdmAppController = this;
 980             }
 981             if (classification != null){
 982                 List<UuidAndTitleCache<TaxonNode>> uuidAndTitleCacheOfAllTaxa = cdmAppController.getClassificationService().getTaxonNodeUuidAndTitleCacheOfAcceptedTaxaByClassification(classification.getUuid());
 983                 if (uuidAndTitleCacheOfAllTaxa != null){
 984                         for (UuidAndTitleCache p : uuidAndTitleCacheOfAllTaxa){
 985                             try{
 986                                 if(p.getTitleCache().equals(taxon.getTitleCache())) {
 987                                     exist = true;
 988                                 }
 989                             }
 990                             catch(Exception e){
 991                                 logger.warn("TaxonNode doesn't seem to have a taxon");
 992                             }
 993                         }
 994                 }
 995             }
 996                 return exist;
 997             }
 998
 999             /**
1000              * join DeterminationEvent to the Taxon Object
1001              * @param state : the ABCD import state
1002              * @param taxon: the current Taxon
1003              * @param preferredFlag :if the current name is preferred
1004              * @param derivedFacade : the derived Unit Facade
1005              */
1006             @SuppressWarnings("rawtypes")
1007         protected void linkDeterminationEvent(STATE state, Taxon taxon, boolean preferredFlag,  DerivedUnitFacade derivedFacade) {
1008                 SpecimenImportConfiguratorBase config = state.getConfig();
1009                 if(DEBUG){
1010                     logger.info("start linkdetermination with taxon:" + taxon.getUuid()+", "+taxon);
1011                 }
1012
1013                 DeterminationEvent determinationEvent = DeterminationEvent.NewInstance();
1014                 determinationEvent.setTaxonName(taxon.getName());
1015                 determinationEvent.setPreferredFlag(preferredFlag);
1016
1017                 determinationEvent.setIdentifiedUnit(state.getDerivedUnitBase());
1018                 state.getDerivedUnitBase().addDetermination(determinationEvent);
1019
1020                 if(DEBUG){
1021                     logger.info("NB TYPES INFO: "+ state.getDataHolder().getStatusList().size());
1022                 }
1023                 for (SpecimenTypeDesignationStatus specimenTypeDesignationstatus : state.getDataHolder().getStatusList()) {
1024                     if (specimenTypeDesignationstatus != null) {
1025                         if(DEBUG){
1026                             logger.info("specimenTypeDesignationstatus :"+ specimenTypeDesignationstatus);
1027                         }
1028
1029                         ICdmApplicationConfiguration cdmAppController = config.getCdmAppController();
1030                         if(cdmAppController == null){
1031                             cdmAppController = this;
1032                         }
1033                         specimenTypeDesignationstatus = (SpecimenTypeDesignationStatus) cdmAppController.getTermService().find(specimenTypeDesignationstatus.getUuid());
1034                         //Designation
1035                         TaxonNameBase<?,?> name = taxon.getName();
1036                         SpecimenTypeDesignation designation = SpecimenTypeDesignation.NewInstance();
1037
1038                         designation.setTypeStatus(specimenTypeDesignationstatus);
1039                         designation.setTypeSpecimen(state.getDerivedUnitBase());
1040                         name.addTypeDesignation(designation, true);
1041                     }
1042                 }
1043
1044                 for (String[] fullReference : state.getDataHolder().getReferenceList()) {
1045
1046
1047                     String strReference=fullReference[0];
1048                     String citationDetail = fullReference[1];
1049                     String citationURL = fullReference[2];
1050                     List<Reference> references = getReferenceService().listByTitle(Reference.class, "strReference", MatchMode.EXACT, null, null, null, null, null);
1051
1052                     if (!references.isEmpty()){
1053                         Reference reference = null;
1054                         for (Reference refe: references) {
1055                             if (refe.getTitleCache().equalsIgnoreCase(strReference)) {
1056                                 reference =refe;
1057                                 break;
1058                             }
1059                         }
1060                         if (reference ==null){
1061                             reference = ReferenceFactory.newGeneric();
1062                             reference.setTitleCache(strReference, true);
1063                             save(reference, state);
1064                         }
1065                         determinationEvent.addReference(reference);
1066                     }
1067                 }
1068                 save(state.getDerivedUnitBase(), state);
1069
1070                 if (config.isAddIndividualsAssociationsSuchAsSpecimenAndObservations() && preferredFlag) {
1071                     //do not add IndividualsAssociation to non-preferred taxa
1072                     if(DEBUG){
1073                         logger.info("isDoCreateIndividualsAssociations");
1074                     }
1075
1076                     makeIndividualsAssociation(state, taxon, determinationEvent);
1077
1078                     save(state.getDerivedUnitBase(), state);
1079                 }
1080             }
1081
1082             /**
1083              * create and link each association (specimen, observation..) to the accepted taxon
1084              * @param state : the ABCD import state
1085              * @param taxon: the current Taxon
1086              * @param determinationEvent:the determinationevent
1087              */
1088             protected void makeIndividualsAssociation(STATE state, Taxon taxon, DeterminationEvent determinationEvent) {
1089                 SpecimenImportConfiguratorBase config = state.getConfig();
1090                 SpecimenUserInteraction sui = config.getSpecimenUserInteraction();
1091
1092                 if (DEBUG) {
1093                     logger.info("MAKE INDIVIDUALS ASSOCIATION");
1094                 }
1095
1096                 TaxonDescription taxonDescription = null;
1097                 Set<TaxonDescription> descriptions= taxon.getDescriptions();
1098 //              if (((Abcd206ImportConfigurator) state.getConfig()).isInteractWithUser()){
1099 //                  if(!state.isDescriptionGroupSet()){
1100 //                      taxonDescription = sui.askForDescriptionGroup(descriptions);
1101 //                      state.setDescriptionGroup(taxonDescription);
1102 //                      state.setDescriptionGroupSet(true);
1103 //                  }else{
1104 //                      taxonDescription=state.getDescriptionGroup();
1105 //                  }
1106 //              } else {
1107                     for (TaxonDescription description : descriptions){
1108                         Set<IdentifiableSource> sources =  new HashSet<>();
1109                         sources.addAll(description.getTaxon().getSources());
1110                         sources.addAll(description.getSources());
1111                         for (IdentifiableSource source:sources){
1112                             if(state.getRef().equals(source.getCitation())) {
1113                                 taxonDescription = description;
1114                             }
1115                         }
1116                     }
1117               //  }
1118                 if (taxonDescription == null){
1119                     taxonDescription = TaxonDescription.NewInstance(taxon, false);
1120                     if(sourceNotLinkedToElement(taxonDescription,state.getRef(),null)) {
1121                         taxonDescription.addSource(OriginalSourceType.Import, null, null, state.getRef(), null);
1122                     }
1123                     state.setDescriptionGroup(taxonDescription);
1124                     taxon.addDescription(taxonDescription);
1125                 }
1126
1127                 //PREPARE REFERENCE QUESTIONS
1128
1129                 Map<String,OriginalSourceBase<?>> sourceMap = new HashMap<String, OriginalSourceBase<?>>();
1130
1131                 List<IdentifiableSource> issTmp = new ArrayList<>();//getCommonService().list(IdentifiableSource.class, null, null, null, null);
1132                 List<DescriptionElementSource> issTmp2 = new ArrayList<>();//getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1133
1134                 Set<OriginalSourceBase> osbSet = new HashSet<OriginalSourceBase>();
1135                 if(issTmp2!=null) {
1136                     osbSet.addAll(issTmp2);
1137                 }
1138                 if(issTmp!=null) {
1139                     osbSet.addAll(issTmp);
1140                 }
1141
1142
1143                 addToSourceMap(sourceMap, osbSet);
1144
1145 //              if (((Abcd206ImportConfigurator) state.getConfig()).isInteractWithUser()){
1146 //                  List<OriginalSourceBase<?>> res = null;
1147 //                  if(!state.isDescriptionSourcesSet()){
1148 //                      res = sui.askForSource(sourceMap, "the description group ("+taxon+")",
1149 //                              "The current reference is "+state.getRef().getTitleCache(),getReferenceService(), state.getDataHolder().docSources);
1150 //                      state.setDescriptionRefs(res);
1151 //                      state.setDescriptionSourcesSet(true);
1152 //                  }
1153 //                  else{
1154 //                      res=state.getDescriptionRefs();
1155 //                  }
1156 //                  if(res !=null) {
1157 //                      for (OriginalSourceBase<?> sour:res){
1158 //                          if(sour.isInstanceOf(IdentifiableSource.class)){
1159 //                              try {
1160 //                                  if(sourceNotLinkedToElement(taxonDescription,sour)) {
1161 //                                      taxonDescription.addSource((IdentifiableSource)sour.clone());
1162 //                                  }
1163 //                              } catch (CloneNotSupportedException e) {
1164 //                                  logger.warn("no cloning?");
1165 //                              }
1166 //                          }else{
1167 //                              if(sourceNotLinkedToElement(taxonDescription,sour)) {
1168 //                                  taxonDescription.addSource(OriginalSourceType.Import,null, null, sour.getCitation(),sour.getCitationMicroReference());
1169 //                              }
1170 //                          }
1171 //                      }
1172 //                  }
1173 //              }
1174 //              else {
1175                     if(sourceNotLinkedToElement(taxonDescription,state.getRef(),null)) {
1176                         taxonDescription.addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1177                     }
1178 //              }
1179                 state.setDescriptionGroup(taxonDescription);
1180
1181                 IndividualsAssociation indAssociation = IndividualsAssociation.NewInstance();
1182                 Feature feature = makeFeature(state.getDerivedUnitBase());
1183                 indAssociation.setAssociatedSpecimenOrObservation(state.getDerivedUnitBase());
1184                 indAssociation.setFeature(feature);
1185 //
1186 //              if (((Abcd206ImportConfigurator) state.getConfig()).isInteractWithUser()){
1187 //                  sourceMap = new HashMap<String, OriginalSourceBase<?>>();
1188 //
1189 //                  issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
1190 //                  issTmp2 = getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1191 //
1192 //                  osbSet = new HashSet<OriginalSourceBase>();
1193 //                  if(issTmp2!=null) {
1194 //                      osbSet.addAll(issTmp2);
1195 //                  }
1196 //                  if(issTmp!=null) {
1197 //                      osbSet.addAll(issTmp);
1198 //                  }
1199 //
1200 //
1201 //                  addToSourceMap(sourceMap, osbSet);
1202 //
1203 //                  List<OriginalSourceBase<?>> sources =null;
1204 //                  if(!state.isAssociationSourcesSet()) {
1205 //                      sources = sui.askForSource(sourceMap,  "descriptive element (association) ",taxon.toString(),
1206 //                              getReferenceService(),state.getDataHolder().getDocSources());
1207 //                      state.setAssociationRefs(sources);
1208 //                      state.setAssociationSourcesSet(true);
1209 //                  }
1210 //                  else{
1211 //                      sources=state.getAssociationRefs();
1212 //                  }
1213 //                  if(sources !=null) {
1214 //                      for (OriginalSourceBase<?> source: sources) {
1215 //                          if(source !=null) {
1216 //                              if(source.isInstanceOf(DescriptionElementSource.class)){
1217 //                                  try {
1218 //                                      if(sourceNotLinkedToElement(indAssociation,source)) {
1219 //                                          indAssociation.addSource((DescriptionElementSource)source.clone());
1220 //                                      }
1221 //                                  } catch (CloneNotSupportedException e) {
1222 //                                      logger.warn("clone forbidden?");
1223 //                                  }
1224 //                              }else{
1225 //                                  if(sourceNotLinkedToElement(indAssociation,source)) {
1226 //                                      indAssociation.addSource(OriginalSourceType.Import,null, null, source.getCitation(),source.getCitationMicroReference());
1227 //                                  }
1228 //                                  try {
1229 //                                      if(sourceNotLinkedToElement(state.getDerivedUnitBase(), source)) {
1230 //                                          state.getDerivedUnitBase().addSource((IdentifiableSource) source.clone());
1231 //                                      }
1232 //                                  } catch (CloneNotSupportedException e) {
1233 //                                      // TODO Auto-generated catch block
1234 //                                      e.printStackTrace();
1235 //                                  }
1236 //                              }
1237 //
1238 //                          }
1239 //                      }
1240 //                  }
1241 //              }else {
1242                     if(sourceNotLinkedToElement(indAssociation,state.getRef(),null)) {
1243                         indAssociation.addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1244                     }
1245                     if(sourceNotLinkedToElement(state.getDerivedUnitBase(), state.getRef(),null)) {
1246                         state.getDerivedUnitBase().addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1247                     }
1248                     for (Reference citation : determinationEvent.getReferences()) {
1249                         if(sourceNotLinkedToElement(indAssociation,citation,null))
1250                         {
1251                             indAssociation.addSource(DescriptionElementSource.NewInstance(OriginalSourceType.Import, null, null, citation, null));
1252                         }
1253                         if(sourceNotLinkedToElement(state.getDerivedUnitBase(), state.getRef(),null)) {
1254                             state.getDerivedUnitBase().addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1255                         }
1256                     }
1257            //     }
1258
1259                 taxonDescription.addElement(indAssociation);
1260
1261                 save(taxonDescription, state);
1262                 save(taxon, state);
1263                 state.getReport().addDerivate(state.getDerivedUnitBase(), config);
1264                 state.getReport().addIndividualAssociation(taxon, state.getDataHolder().getUnitID(), state.getDerivedUnitBase());
1265             }
1266
1267             /**
1268              * @param derivedUnitBase2
1269              * @param ref2
1270              * @param object
1271              * @return
1272              */
1273             private boolean sourceNotLinkedToElement(DerivedUnit derivedUnitBase2, Reference b, String d) {
1274                 Set<IdentifiableSource> linkedSources = derivedUnitBase2.getSources();
1275                 for (IdentifiableSource is:linkedSources){
1276                     Reference a = is.getCitation();
1277                     String c = is.getCitationMicroReference();
1278
1279                     boolean refMatch=false;
1280                     boolean microMatch=false;
1281
1282                     try{
1283                         if (a==null && b==null) {
1284                             refMatch=true;
1285                         }
1286                         if (a!=null && b!=null) {
1287                             if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1288                                 refMatch=true;
1289                             }
1290                         }
1291                     }catch(Exception e){}
1292
1293
1294                     try{
1295                         if (c==null && d==null) {
1296                             microMatch=true;
1297                         }
1298                         if(c!=null && d!=null) {
1299                             if(c.equalsIgnoreCase(d)) {
1300                                 microMatch=true;
1301                             }
1302                         }
1303                     }
1304                     catch(Exception e){}
1305
1306                     if (microMatch && refMatch) {
1307                         return false;
1308                     }
1309
1310
1311                 }
1312                 return true;
1313             }
1314
1315             private <T extends OriginalSourceBase<?>> boolean  sourceNotLinkedToElement(ISourceable<T> sourcable, Reference reference, String microReference) {
1316                 Set<T> linkedSources = sourcable.getSources();
1317                 for (T is:linkedSources){
1318                     Reference unitReference = is.getCitation();
1319                     String unitMicroReference = is.getCitationMicroReference();
1320
1321                     boolean refMatch=false;
1322                     boolean microMatch=false;
1323
1324                     try{
1325                         if (unitReference==null && reference==null) {
1326                             refMatch=true;
1327                         }
1328                         if (unitReference!=null && reference!=null) {
1329                             if (unitReference.getTitleCache().equalsIgnoreCase(reference.getTitleCache())) {
1330                                 refMatch=true;
1331                             }
1332                         }
1333                     }catch(Exception e){}
1334
1335                     try{
1336                         if (unitMicroReference==null && microReference==null) {
1337                             microMatch=true;
1338                         }
1339                         if(unitMicroReference!=null && microReference!=null) {
1340                             if(unitMicroReference.equalsIgnoreCase(microReference)) {
1341                                 microMatch=true;
1342                             }
1343                         }
1344                     }
1345                     catch(Exception e){}
1346
1347                     if (microMatch && refMatch) {
1348                         return false;
1349                     }
1350                 }
1351                 return true;
1352             }
1353
1354             /**
1355              * look for the Feature object (FieldObs, Specimen,...)
1356              * @param unit : a specimen or obersvation base
1357              * @return the corresponding Feature
1358              */
1359             private Feature makeFeature(SpecimenOrObservationBase<?> unit) {
1360                 SpecimenOrObservationType type = unit.getRecordBasis();
1361
1362
1363
1364                 if (type.isFeatureObservation()){
1365                     return Feature.OBSERVATION();
1366                 }else if (type.isFeatureSpecimen()){
1367                     return Feature.SPECIMEN();
1368                 }else if (type == SpecimenOrObservationType.DerivedUnit){
1369                     return Feature.OBSERVATION();
1370                     //            return getFeature("Specimen or observation");
1371                 }else{
1372                     String message = "Unhandled record basis '%s' for defining individuals association feature type. Use default.";
1373                     logger.warn(String.format(message, type.getMessage()));
1374                     return Feature.OBSERVATION();
1375                     //            return getFeature("Specimen or observation");
1376
1377                 }
1378             }
1379
1380
1381             /**
1382              * @param sourceMap
1383              * @param osbSet
1384              */
1385             protected void addToSourceMap(Map<String, OriginalSourceBase<?>> sourceMap, Set<OriginalSourceBase> osbSet) {
1386                 for( OriginalSourceBase<?> osb:osbSet) {
1387                     if(osb.getCitation()!=null && osb.getCitationMicroReference() !=null  && !osb.getCitationMicroReference().isEmpty()) {
1388                         try{
1389                             sourceMap.put(osb.getCitation().getTitleCache()+ "---"+osb.getCitationMicroReference(),osb);
1390                         }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1391                     } else if(osb.getCitation()!=null){
1392                         try{
1393                             sourceMap.put(osb.getCitation().getTitleCache(),osb);
1394                         }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1395                     }
1396                 }
1397             }
1398
1399 }