cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/SpecimenImportBase.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.specimen;
  11
  12
  13 import java.util.ArrayList;
  14 import java.util.HashMap;
  15 import java.util.HashSet;
  16 import java.util.List;
  17 import java.util.Map;
  18 import java.util.Set;
  19 import java.util.UUID;
  20
  21 import org.apache.log4j.Logger;
  22
  23 import eu.etaxonomy.cdm.api.application.ICdmApplicationConfiguration;
  24 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
  25 import eu.etaxonomy.cdm.api.service.config.FindOccurrencesConfigurator;
  26 import eu.etaxonomy.cdm.api.service.pager.Pager;
  27 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  28 import eu.etaxonomy.cdm.io.common.CdmImportBase;
  29 import eu.etaxonomy.cdm.io.common.IImportConfigurator;
  30 import eu.etaxonomy.cdm.io.specimen.abcd206.in.Identification;
  31 import eu.etaxonomy.cdm.io.specimen.abcd206.in.SpecimenImportReport;
  32 import eu.etaxonomy.cdm.model.agent.AgentBase;
  33 import eu.etaxonomy.cdm.model.agent.Institution;
  34 import eu.etaxonomy.cdm.model.agent.Team;
  35 import eu.etaxonomy.cdm.model.common.CdmBase;
  36 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  37 import eu.etaxonomy.cdm.model.common.LanguageString;
  38 import eu.etaxonomy.cdm.model.common.OriginalSourceBase;
  39 import eu.etaxonomy.cdm.model.common.OriginalSourceType;
  40 import eu.etaxonomy.cdm.model.description.DescriptionBase;
  41 import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
  42 import eu.etaxonomy.cdm.model.description.Feature;
  43 import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
  44 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  45 import eu.etaxonomy.cdm.model.name.BacterialName;
  46 import eu.etaxonomy.cdm.model.name.BotanicalName;
  47 import eu.etaxonomy.cdm.model.name.CultivarPlantName;
  48 import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
  49 import eu.etaxonomy.cdm.model.name.NonViralName;
  50 import eu.etaxonomy.cdm.model.name.Rank;
  51 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
  52 import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
  53 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
  54 import eu.etaxonomy.cdm.model.name.ZoologicalName;
  55 import eu.etaxonomy.cdm.model.occurrence.Collection;
  56 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  57 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
  58 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
  59 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationType;
  60 import eu.etaxonomy.cdm.model.reference.Reference;
  61 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  62 import eu.etaxonomy.cdm.model.taxon.Classification;
  63 import eu.etaxonomy.cdm.model.taxon.Synonym;
  64 import eu.etaxonomy.cdm.model.taxon.Taxon;
  65 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  66 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  67 import eu.etaxonomy.cdm.persistence.dto.UuidAndTitleCache;
  68 import eu.etaxonomy.cdm.persistence.query.MatchMode;
  69 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
  70 import eu.etaxonomy.cdm.strategy.parser.ParserProblem;
  71
  72
  73 /**
  74  * @author p.kelbert
  75  * @created 20.10.2008
  76  */
  77 public abstract class SpecimenImportBase<CONFIG extends IImportConfigurator, STATE extends SpecimenImportStateBase>  extends CdmImportBase<CONFIG, STATE> {
  78
  79         @SuppressWarnings("unused")
  80         private static final Logger logger = Logger.getLogger(SpecimenImportBase.class);
  81         protected final boolean DEBUG = true;
  82
  83         protected static final UUID SPECIMEN_SCAN_TERM = UUID.fromString("acda15be-c0e2-4ea8-8783-b9b0c4ad7f03");
  84
  85         private static final String COLON = ":";
  86
  87
  88         @Override
  89     protected abstract void doInvoke(STATE state);
  90
  91         /**
  92          * Handle a single unit
  93          * @param state
  94          * @param item
  95          */
  96         @SuppressWarnings("rawtypes")
  97         protected abstract void handleSingleUnit(STATE state, Object item) ;
  98
  99
 100
 101         protected TaxonNameBase<?, ?> getOrCreateTaxonName(String scientificName, Rank rank, boolean preferredFlag, STATE state, int unitIndexInAbcdFile){
 102         TaxonNameBase<?, ?> taxonName = null;
 103         SpecimenImportConfiguratorBase config = state.getConfig();
 104
 105         //check atomised name data for rank
 106         //new name will be created
 107         NonViralName<?> atomisedTaxonName = null;
 108         if (rank==null && unitIndexInAbcdFile>=0 && ((state.getDataHolder().getAtomisedIdentificationList() != null && !state.getDataHolder().getAtomisedIdentificationList().isEmpty())|| state.getDataHolder().getAtomisedIdentificationList().size() > 0)) {
 109             atomisedTaxonName = setTaxonNameByType(state.getDataHolder().getAtomisedIdentificationList().get(unitIndexInAbcdFile), scientificName, state);
 110             if(atomisedTaxonName!=null){
 111                 rank = atomisedTaxonName.getRank();
 112             }
 113         }
 114         if(config.isReuseExistingTaxaWhenPossible()){
 115             NonViralName<?> parsedName = atomisedTaxonName;
 116             if(parsedName==null){
 117                 parsedName = parseScientificName(scientificName, state, state.getReport());
 118             }
 119             if(config.isIgnoreAuthorship() && parsedName!=null && preferredFlag){
 120                 // do not ignore authorship for non-preferred names because they need
 121                 // to be created for the determination history
 122                 String nameCache = parsedName.getNameCache();
 123                 List<NonViralName> names = getNameService().findNamesByNameCache(nameCache, MatchMode.EXACT, null);
 124                 if (!names.isEmpty()){
 125                      return getBestMatchingName(scientificName, new ArrayList<TaxonNameBase>(names), state);
 126                 }
 127             } else {
 128                 //search for existing names
 129                 List<TaxonNameBase> names = getNameService().listByTitle(TaxonNameBase.class, scientificName, MatchMode.EXACT, null, null, null, null, null);
 130                 taxonName = getBestMatchingName(scientificName, names, state);
 131                 //still nothing found -> try with the atomised name full title cache
 132                 if(taxonName==null && atomisedTaxonName!=null){
 133                     names = getNameService().listByTitle(TaxonNameBase.class, atomisedTaxonName.getFullTitleCache(), MatchMode.EXACT, null, null, null, null, null);
 134                     taxonName = getBestMatchingName(atomisedTaxonName.getTitleCache(), names, state);
 135                     //still nothing found -> try with the atomised name title cache
 136                     if(taxonName==null){
 137                         names = getNameService().listByTitle(TaxonNameBase.class, atomisedTaxonName.getTitleCache(), MatchMode.EXACT, null, null, null, null, null);
 138                         taxonName = getBestMatchingName(atomisedTaxonName.getTitleCache(), names, state);
 139                     }
 140                 }
 141
 142             }
 143
 144         }
 145
 146         if(taxonName==null && atomisedTaxonName!=null){
 147             taxonName = atomisedTaxonName;
 148             state.getReport().addName(taxonName);
 149             logger.info("Created new taxon name "+taxonName);
 150             if(taxonName.hasProblem()){
 151                 state.getReport().addInfoMessage(String.format("Created %s with parsing problems", taxonName));
 152             }
 153             if(!atomisedTaxonName.getTitleCache().equals(scientificName)){
 154                 state.getReport().addInfoMessage(String.format("Taxon %s was parsed as %s", scientificName, atomisedTaxonName.getTitleCache()));
 155             }
 156         }
 157         else if(taxonName==null){
 158             //create new taxon name
 159             if (state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNAFP)){
 160                 taxonName = BotanicalName.NewInstance(rank);
 161             }else if (state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICZN)){
 162                 taxonName = ZoologicalName.NewInstance(rank);
 163             }else{
 164                 taxonName = NonViralName.NewInstance(rank);
 165             }
 166             taxonName.setFullTitleCache(scientificName,true);
 167             taxonName.setTitleCache(scientificName, true);
 168             state.getReport().addName(taxonName);
 169             logger.info("Created new taxon name "+taxonName);
 170         }
 171         save(taxonName, state);
 172         return taxonName;
 173     }
 174
 175          protected TaxonNameBase<?, ?> getBestMatchingName(String scientificName, java.util.Collection<TaxonNameBase> names, STATE state){
 176                 List<TaxonNameBase> namesWithAcceptedTaxa = new ArrayList<TaxonNameBase>();
 177                 for (TaxonNameBase name : names) {
 178                     if(!name.getTaxa().isEmpty()){
 179                         namesWithAcceptedTaxa.add(name);
 180                     }
 181                 }
 182                 String message = "More than one taxon name was found for "+scientificName+"!";
 183                 //check for names with accepted taxa
 184                 if(namesWithAcceptedTaxa.size()>0){
 185                     if(namesWithAcceptedTaxa.size()>1){
 186                         state.getReport().addInfoMessage(message);
 187                         logger.warn(message);
 188                         return null;
 189                     }
 190                     return namesWithAcceptedTaxa.iterator().next();
 191                 }
 192                 //no names with accepted taxa found -> check accepted taxa of synonyms
 193                 List<Taxon> taxaFromSynonyms = new ArrayList<Taxon>();
 194                 for (TaxonNameBase name : names) {
 195                     Set<TaxonBase> taxonBases = name.getTaxonBases();
 196                     for (TaxonBase taxonBase : taxonBases) {
 197                         if(taxonBase.isInstanceOf(Synonym.class)){
 198                             Synonym synonym = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
 199                             taxaFromSynonyms.addAll(synonym.getAcceptedTaxa());
 200                         }
 201                     }
 202                 }
 203                 if(taxaFromSynonyms.size()>0){
 204                     if(taxaFromSynonyms.size()>1){
 205                         state.getReport().addInfoMessage(message);
 206                         logger.warn(message);
 207                         return null;
 208                     }
 209                     return taxaFromSynonyms.iterator().next().getName();
 210                 }
 211                 return null;
 212             }
 213          /**
 214              * Parse automatically the scientific name
 215              * @param scientificName the scientific name to parse
 216              * @param state the current import state
 217              * @param report the import report
 218              * @return a parsed name
 219              */
 220             protected NonViralName<?> parseScientificName(String scientificName, STATE state, SpecimenImportReport report) {
 221                 NonViralNameParserImpl nvnpi = NonViralNameParserImpl.NewInstance();
 222                 NonViralName<?> taxonName = null;
 223                 boolean problem = false;
 224
 225                 if(DEBUG){
 226                     logger.info("parseScientificName " + state.getDataHolder().getNomenclatureCode().toString());
 227                 }
 228
 229                 if (state.getDataHolder().getNomenclatureCode().toString().equals("Zoological") || state.getDataHolder().getNomenclatureCode().toString().contains("ICZN")) {
 230                     taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICZN, null);
 231                     if (taxonName.hasProblem()) {
 232                         problem = true;
 233                     }
 234                 }
 235                 else if (state.getDataHolder().getNomenclatureCode().toString().equals("Botanical") || state.getDataHolder().getNomenclatureCode().toString().contains("ICBN")) {
 236                     taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNAFP, null);
 237                     if (taxonName.hasProblem()) {
 238                         problem = true;
 239                     }
 240                 }
 241                 else if (state.getDataHolder().getNomenclatureCode().toString().equals("Bacterial") || state.getDataHolder().getNomenclatureCode().toString().contains("ICBN")) {
 242                     taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNB, null);
 243                     if (taxonName.hasProblem()) {
 244                         problem = true;
 245                     }
 246                 }
 247                 else if (state.getDataHolder().getNomenclatureCode().toString().equals("Cultivar") || state.getDataHolder().getNomenclatureCode().toString().contains("ICNCP")) {
 248                     taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNCP, null);
 249                     if (taxonName.hasProblem()) {
 250                         problem = true;
 251                     }
 252                 }
 253                 if (problem) {
 254                     String message = String.format("Parsing problems for %s", scientificName);
 255                     if(taxonName!=null){
 256                         for (ParserProblem parserProblem : taxonName.getParsingProblems()) {
 257                             message += "\n\t- "+parserProblem;
 258                         }
 259                     }
 260                     report.addInfoMessage(message);
 261                     logger.info(message);
 262                 }
 263                 return taxonName;
 264
 265             }
 266
 267             /**
 268              * Create the name without automatic parsing, either because it failed, or because the user deactivated it.
 269              * The name is built upon the ABCD fields
 270              * @param atomisedMap : the ABCD atomised fields
 271              * @param fullName : the full scientific name
 272              * @param state
 273              * @return the corresponding Botanical or Zoological or... name
 274              */
 275             protected NonViralName<?> setTaxonNameByType(
 276                     HashMap<String, String> atomisedMap, String fullName, STATE state) {
 277                 boolean problem = false;
 278                 if(DEBUG) {
 279                     logger.info("settaxonnamebytype " + state.getDataHolder().getNomenclatureCode().toString());
 280                 }
 281
 282                 if (state.getDataHolder().getNomenclatureCode().equals("Zoological")) {
 283                     NonViralName<ZoologicalName> taxonName = ZoologicalName.NewInstance(null);
 284                     taxonName.setFullTitleCache(fullName, true);
 285                     taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
 286                     taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
 287                     taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap,"SpeciesEpithet")));
 288                     taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap,"SubspeciesEpithet")));
 289
 290                     if (taxonName.getGenusOrUninomial() != null){
 291                         taxonName.setRank(Rank.GENUS());
 292                     }
 293
 294                     if (taxonName.getInfraGenericEpithet() != null){
 295                         taxonName.setRank(Rank.SUBGENUS());
 296                     }
 297
 298                     if (taxonName.getSpecificEpithet() != null){
 299                         taxonName.setRank(Rank.SPECIES());
 300                     }
 301
 302                     if (taxonName.getInfraSpecificEpithet() != null){
 303                         taxonName.setRank(Rank.SUBSPECIES());
 304                     }
 305
 306                     Team team = null;
 307                     if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 308                         team = Team.NewInstance();
 309                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
 310                     }
 311                     else {
 312                         if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 313                             team = Team.NewInstance();
 314                             team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
 315                         }
 316                     }
 317                     if (team != null) {
 318                         taxonName.setBasionymAuthorship(team);
 319                     }
 320                     else {
 321                         if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 322                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
 323                         }
 324                         else if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 325                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamAndYear"));
 326                         }
 327                     }
 328                     if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
 329                         team = Team.NewInstance();
 330                         team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
 331                         taxonName.setCombinationAuthorship(team);
 332                     }
 333                     if (taxonName.hasProblem()) {
 334                         logger.info("pb ICZN");
 335                         problem = true;
 336                     }
 337                     else {
 338                         return taxonName;
 339                     }
 340                 }
 341                 else if (state.getDataHolder().getNomenclatureCode().equals("Botanical")) {
 342                     BotanicalName taxonName = (BotanicalName) parseScientificName(fullName, state, state.getReport());
 343                     if (taxonName != null){
 344                         return taxonName;
 345                     }
 346                     else{
 347                         taxonName = BotanicalName.NewInstance(null);
 348                     }
 349                     taxonName.setFullTitleCache(fullName, true);
 350                     taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
 351                     taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "FirstEpithet")));
 352                     taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "InfraSpeEpithet")));
 353                     try {
 354                         taxonName.setRank(Rank.getRankByName(getFromMap(atomisedMap, "Rank")));
 355                     } catch (Exception e) {
 356                         if (taxonName.getInfraSpecificEpithet() != null){
 357                             taxonName.setRank(Rank.SUBSPECIES());
 358                         }
 359                         else if (taxonName.getSpecificEpithet() != null){
 360                             taxonName.setRank(Rank.SPECIES());
 361                         }
 362                         else if (taxonName.getInfraGenericEpithet() != null){
 363                             taxonName.setRank(Rank.SUBGENUS());
 364                         }
 365                         else if (taxonName.getGenusOrUninomial() != null){
 366                             taxonName.setRank(Rank.GENUS());
 367                         }
 368                     }
 369                     Team team = null;
 370                     if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 371                         team = Team.NewInstance();
 372                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
 373                         taxonName.setBasionymAuthorship(team);
 374                     }
 375                     if (getFromMap(atomisedMap, "AuthorTeam") != null) {
 376                         team = Team.NewInstance();
 377                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeam"), true);
 378                         taxonName.setCombinationAuthorship(team);
 379                     }
 380                     if (team == null) {
 381                         if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
 382                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
 383                         }
 384                         else if (getFromMap(atomisedMap, "AuthorTeam") != null) {
 385                             taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeam"));
 386                         }
 387                     }
 388                     if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
 389                         team = Team.NewInstance();
 390                         team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
 391                         taxonName.setCombinationAuthorship(team);
 392                     }
 393                     if (taxonName.hasProblem()) {
 394                         logger.info("pb ICBN");
 395                         problem = true;
 396                     }
 397                     else {
 398                         return taxonName;
 399                     }
 400                 }
 401                 else if (state.getDataHolder().getNomenclatureCode().equals("Bacterial")) {
 402                     NonViralName<BacterialName> taxonName = BacterialName.NewInstance(null);
 403                     taxonName.setFullTitleCache(fullName, true);
 404                     taxonName.setGenusOrUninomial(getFromMap(atomisedMap, "Genus"));
 405                     taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
 406                     taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "Species")));
 407                     taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "SubspeciesEpithet")));
 408
 409                     if (taxonName.getGenusOrUninomial() != null){
 410                         taxonName.setRank(Rank.GENUS());
 411                     }
 412                     else if (taxonName.getInfraGenericEpithet() != null){
 413                         taxonName.setRank(Rank.SUBGENUS());
 414                     }
 415                     else if (taxonName.getSpecificEpithet() != null){
 416                         taxonName.setRank(Rank.SPECIES());
 417                     }
 418                     else if (taxonName.getInfraSpecificEpithet() != null){
 419                         taxonName.setRank(Rank.SUBSPECIES());
 420                     }
 421
 422                     if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
 423                         Team team = Team.NewInstance();
 424                         team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
 425                         taxonName.setCombinationAuthorship(team);
 426                     }
 427                     if (getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear") != null) {
 428                         Team team = Team.NewInstance();
 429                         team.setTitleCache(getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear"), true);
 430                         taxonName.setBasionymAuthorship(team);
 431                     }
 432                     if (taxonName.hasProblem()) {
 433                         logger.info("pb ICNB");
 434                         problem = true;
 435                     }
 436                     else {
 437                         return taxonName;
 438                     }
 439                 }
 440                 else if (state.getDataHolder().getNomenclatureCode().equals("Cultivar")) {
 441                     CultivarPlantName taxonName = CultivarPlantName.NewInstance(null);
 442
 443                     if (taxonName.hasProblem()) {
 444                         logger.info("pb ICNCP");
 445                         problem = true;
 446                     }
 447                     else {
 448                         return taxonName;
 449                     }
 450                     return taxonName;
 451                 }
 452
 453                 if (problem) {
 454                     logger.info("Problem im setTaxonNameByType ");
 455                     NonViralName<?> taxonName = NonViralName.NewInstance(null);
 456                     taxonName.setFullTitleCache(fullName, true);
 457                     return taxonName;
 458                 }
 459                 NonViralName<?> tn = NonViralName.NewInstance(null);
 460                 return tn;
 461             }
 462
 463             /**
 464              * Get a formated string from a hashmap
 465              * @param atomisedMap
 466              * @param key
 467              * @return
 468              */
 469             private String getFromMap(HashMap<String, String> atomisedMap, String key) {
 470                 String value = null;
 471                 if (atomisedMap.containsKey(key)) {
 472                     value = atomisedMap.get(key);
 473                 }
 474
 475                 try {
 476                     if (value != null && key.matches(".*Year.*")) {
 477                         value = value.trim();
 478                         if (value.matches("[a-z A-Z ]*[0-9]{4}$")) {
 479                             String tmp = value.split("[0-9]{4}$")[0];
 480                             int year = Integer.parseInt(value.split(tmp)[1]);
 481                             if (year >= 1752) {
 482                                 value = tmp;
 483                             }
 484                             else {
 485                                 value = null;
 486                             }
 487                         }
 488                         else {
 489                             value = null;
 490                         }
 491                     }
 492                 }
 493                 catch (Exception e) {
 494                     value = null;
 495                 }
 496                 return value;
 497             }
 498
 499             /**
 500              * Very fast and dirty implementation to allow handling of transient objects as described in
 501              * https://dev.e-taxonomy.eu/trac/ticket/3726
 502              *
 503              * Not yet complete.
 504              *
 505              * @param cdmBase
 506              * @param state
 507              */
 508             protected UUID save(CdmBase cdmBase, SpecimenImportStateBase state) {
 509                 ICdmApplicationConfiguration cdmRepository = state.getConfig().getCdmAppController();
 510                 if (cdmRepository == null){
 511                     cdmRepository = this;
 512                 }
 513
 514                 if (cdmBase.isInstanceOf(LanguageString.class)){
 515                     return cdmRepository.getTermService().saveLanguageData(CdmBase.deproxy(cdmBase, LanguageString.class));
 516                 }else if (cdmBase.isInstanceOf(SpecimenOrObservationBase.class)){
 517                     return cdmRepository.getOccurrenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, SpecimenOrObservationBase.class));
 518                 }else if (cdmBase.isInstanceOf(Reference.class)){
 519                     return cdmRepository.getReferenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, Reference.class));
 520                 }else if (cdmBase.isInstanceOf(Classification.class)){
 521                     return cdmRepository.getClassificationService().saveOrUpdate(CdmBase.deproxy(cdmBase, Classification.class));
 522                 }else if (cdmBase.isInstanceOf(AgentBase.class)){
 523                     return cdmRepository.getAgentService().saveOrUpdate(CdmBase.deproxy(cdmBase, AgentBase.class));
 524                 }else if (cdmBase.isInstanceOf(Collection.class)){
 525                     return cdmRepository.getCollectionService().saveOrUpdate(CdmBase.deproxy(cdmBase, Collection.class));
 526                 }else if (cdmBase.isInstanceOf(DescriptionBase.class)){
 527                     return cdmRepository.getDescriptionService().saveOrUpdate(CdmBase.deproxy(cdmBase, DescriptionBase.class));
 528                 }else if (cdmBase.isInstanceOf(TaxonBase.class)){
 529                     return cdmRepository.getTaxonService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonBase.class));
 530                 }else if (cdmBase.isInstanceOf(TaxonNameBase.class)){
 531                     return cdmRepository.getNameService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonNameBase.class));
 532                 }else{
 533                     throw new IllegalArgumentException("Class not supported in save method: " + CdmBase.deproxy(cdmBase, CdmBase.class).getClass().getSimpleName());
 534                 }
 535
 536             }
 537
 538
 539             protected SpecimenOrObservationBase findExistingSpecimen(String unitId, SpecimenImportStateBase state){
 540                 ICdmApplicationConfiguration cdmAppController = state.getConfig().getCdmAppController();
 541                 if(cdmAppController==null){
 542                     cdmAppController = this;
 543                 }
 544                 FindOccurrencesConfigurator config = new FindOccurrencesConfigurator();
 545                 config.setSignificantIdentifier(unitId);
 546                 Pager<SpecimenOrObservationBase> existingSpecimens = cdmAppController.getOccurrenceService().findByTitle(config);
 547                 if(!existingSpecimens.getRecords().isEmpty()){
 548                     if(existingSpecimens.getRecords().size()==1){
 549                         return existingSpecimens.getRecords().iterator().next();
 550                     }
 551                 }
 552                 return null;
 553             }
 554
 555             protected abstract void importAssociatedUnits(STATE state, Object item, DerivedUnitFacade derivedUnitFacade);
 556
 557             /**
 558              * getFacade : get the DerivedUnitFacade based on the recordBasis
 559              * @param state
 560              *
 561              * @return DerivedUnitFacade
 562              */
 563             protected DerivedUnitFacade getFacade(STATE state) {
 564                 if(DEBUG) {
 565                     logger.info("getFacade()");
 566                 }
 567                 SpecimenOrObservationType type = null;
 568
 569                 // create specimen
 570                 if (NB((state.getDataHolder().getRecordBasis())) != null) {
 571                     if (state.getDataHolder().getRecordBasis().toLowerCase().startsWith("s") || state.getDataHolder().getRecordBasis().toLowerCase().indexOf("specimen")>-1) {// specimen
 572                         type = SpecimenOrObservationType.PreservedSpecimen;
 573                     }
 574                     if (state.getDataHolder().getRecordBasis().toLowerCase().startsWith("o") ||state.getDataHolder().getRecordBasis().toLowerCase().indexOf("observation")>-1 ) {
 575                         type = SpecimenOrObservationType.Observation;
 576                     }
 577                     if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("fossil")>-1){
 578                         type = SpecimenOrObservationType.Fossil;
 579                     }
 580                     if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("living")>-1) {
 581                         type = SpecimenOrObservationType.LivingSpecimen;
 582                     }
 583                     if (type == null) {
 584                         logger.info("The basis of record does not seem to be known: " + state.getDataHolder().getRecordBasis());
 585                         type = SpecimenOrObservationType.DerivedUnit;
 586                     }
 587                     // TODO fossils?
 588                 } else {
 589                     logger.info("The basis of record is null");
 590                     type = SpecimenOrObservationType.DerivedUnit;
 591                 }
 592                 DerivedUnitFacade derivedUnitFacade = DerivedUnitFacade.NewInstance(type);
 593                 return derivedUnitFacade;
 594             }
 595
 596             /**
 597              * Look if the Institution does already exist
 598              * @param institutionCode: a string with the institutioncode
 599              * @param config : the configurator
 600              * @return the Institution (existing or new)
 601              */
 602             protected Institution getInstitution(String institutionCode, STATE state) {
 603                 SpecimenImportConfiguratorBase config = state.getConfig();
 604                 Institution institution=null;
 605                 institution = (Institution)state.institutions.get(institutionCode);
 606                 if (institution != null){
 607                     return institution;
 608                 }
 609                 List<Institution> institutions;
 610                 try {
 611                     institutions = getAgentService().searchInstitutionByCode(institutionCode);
 612
 613                 } catch (Exception e) {
 614                     institutions = new ArrayList<Institution>();
 615                     logger.warn(e);
 616                 }
 617                 if (institutions.size() > 0 && config.isReuseExistingMetaData()) {
 618                     for (Institution institut:institutions){
 619                         try{
 620                             if (institut.getCode().equalsIgnoreCase(institutionCode)) {
 621                                 institution=institut;
 622                                 break;
 623                             }
 624                         }catch(Exception e){logger.warn("no institution code in the db");}
 625                     }
 626                 }
 627                 if(DEBUG) {
 628                     if(institution !=null) {
 629                         logger.info("getinstitution " + institution.toString());
 630                     }
 631                 }
 632                 if (institution == null){
 633                     // create institution
 634                     institution = Institution.NewInstance();
 635                     institution.setCode(institutionCode);
 636                     institution.setTitleCache(institutionCode, true);
 637                     UUID uuid = save(institution, state);
 638                 }
 639
 640
 641                 state.institutions.put(institutionCode, institution);
 642                 return institution;
 643             }
 644
 645             /**
 646              * Look if the Collection does already exist
 647              * @param collectionCode
 648              * @param collectionCode: a string
 649              * @param config : the configurator
 650              * @return the Collection (existing or new)
 651              */
 652             protected Collection getCollection(Institution institution, String collectionCode, STATE state) {
 653                 SpecimenImportConfiguratorBase config = state.getConfig();
 654                 Collection collection = null;
 655                 List<Collection> collections;
 656                 collection = (Collection) state.collections.get(collectionCode);
 657                 if (collection != null){
 658                     return collection;
 659                 }
 660                 try {
 661                     collections = getCollectionService().searchByCode(collectionCode);
 662                 } catch (Exception e) {
 663                     collections = new ArrayList<Collection>();
 664                 }
 665                 if (collections.size() > 0 && config.isReuseExistingMetaData()) {
 666                     for (Collection coll:collections){
 667                         if (coll.getCode() != null && coll.getInstitute() != null
 668                                 && coll.getCode().equalsIgnoreCase(collectionCode) && coll.getInstitute().equals(institution)) {
 669                             collection = coll;
 670                             break;
 671                         }
 672                     }
 673                 }
 674
 675                 if(collection == null){
 676                     collection =Collection.NewInstance();
 677                     collection.setCode(collectionCode);
 678                     collection.setInstitute(institution);
 679                     collection.setTitleCache(collectionCode);
 680                     UUID uuid = save(collection, state);
 681                 }
 682
 683
 684
 685                 state.collections.put(collectionCode, collection);
 686
 687                 return collection;
 688             }
 689
 690             /**
 691              * @param reference
 692              * @param citationDetail
 693              * @return
 694              */
 695             //FIXME this method is highly critical, because
 696             //  * it will have serious performance and memory problems with large databases
 697             //        (databases may easily have >1 Mio source records)
 698             //  * it does not make sense to search for existing sources and then clone them
 699             //    we need to search for existing references instead and use them (if exist)
 700             //    for our new source.
 701             protected IdentifiableSource getIdentifiableSource(Reference reference, String citationDetail) {
 702
 703               /*  List<IdentifiableSource> issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
 704
 705
 706                 if (reference != null){
 707                     try {
 708                         for (OriginalSourceBase<?> osb: issTmp){
 709                             if (osb.getCitation() != null && osb.getCitation().getTitleCache().equalsIgnoreCase(reference.getTitleCache())){
 710                                 String osbDetail = osb.getCitationMicroReference();
 711                                 if ((StringUtils.isBlank(osbDetail) && StringUtils.isBlank(citationDetail))
 712                                         || (osbDetail != null && osbDetail.equalsIgnoreCase(citationDetail)) ) {
 713 //                                  System.out.println("REFERENCE FOUND RETURN EXISTING SOURCE");
 714                                     return (IdentifiableSource) osb.clone();
 715                                 }
 716                             }
 717                         }
 718                     } catch (CloneNotSupportedException e) {
 719                         throw new RuntimeException(e);
 720                     } catch (Exception e1){
 721                         e1.printStackTrace();
 722                     }
 723                 }
 724         */
 725                 IdentifiableSource sour = IdentifiableSource.NewInstance(OriginalSourceType.Import,null,null, reference,citationDetail);
 726                 return sour;
 727             }
 728
 729             /**
 730              * Add the hierarchy for a Taxon(add higher taxa)
 731              * @param classification
 732              * @param taxon: a taxon to add as a node
 733              * @param state: the ABCD import state
 734              */
 735             protected void addParentTaxon(Taxon taxon, STATE state, boolean preferredFlag, Classification classification){
 736                 NonViralName<?>  nvname = CdmBase.deproxy(taxon.getName(), NonViralName.class);
 737                 Rank rank = nvname.getRank();
 738                 Taxon genus =null;
 739                 Taxon subgenus =null;
 740                 Taxon species = null;
 741                 Taxon subspecies = null;
 742                 Taxon parent = null;
 743                 if(rank!=null){
 744                     if (rank.isLower(Rank.GENUS() )){
 745                         String genusOrUninomial = nvname.getGenusOrUninomial();
 746                         TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(genusOrUninomial, Rank.GENUS(), preferredFlag, state, -1);
 747                         genus = getOrCreateTaxonForName(taxonName, state);
 748                         if (preferredFlag) {
 749                             parent = linkParentChildNode(null, genus, classification, state);
 750                         }
 751
 752                     }
 753                     if (rank.isLower(Rank.SUBGENUS())){
 754                         String prefix = nvname.getGenusOrUninomial();
 755                         String name = nvname.getInfraGenericEpithet();
 756                         if (name != null){
 757                             TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(prefix+" "+name, Rank.SUBGENUS(), preferredFlag, state, -1);
 758                             subgenus = getOrCreateTaxonForName(taxonName, state);
 759                             if (preferredFlag) {
 760                                 parent = linkParentChildNode(genus, subgenus, classification, state);
 761                             }            }
 762                     }
 763                     if (rank.isLower(Rank.SPECIES())){
 764                         if (subgenus!=null){
 765                             String prefix = nvname.getGenusOrUninomial();
 766                             String name = nvname.getInfraGenericEpithet();
 767                             String spe = nvname.getSpecificEpithet();
 768                             if (spe != null){
 769                                 TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(prefix+" "+name+" "+spe, Rank.SPECIES(), preferredFlag, state, -1);
 770                                 species = getOrCreateTaxonForName(taxonName, state);
 771                                 if (preferredFlag) {
 772                                     parent = linkParentChildNode(subgenus, species, classification, state);
 773                                 }
 774                             }
 775                         }
 776                         else{
 777                             String prefix = nvname.getGenusOrUninomial();
 778                             String name = nvname.getSpecificEpithet();
 779                             if (name != null){
 780                                 TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(prefix+" "+name, Rank.SPECIES(), preferredFlag, state, -1);
 781                                 species = getOrCreateTaxonForName(taxonName, state);
 782                                 if (preferredFlag) {
 783                                     parent = linkParentChildNode(genus, species, classification, state);
 784                                 }
 785                             }
 786                         }
 787                     }
 788                     if (rank.isLower(Rank.INFRASPECIES())){
 789                         TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(nvname.getFullTitleCache(), Rank.SUBSPECIES(), preferredFlag, state, -1);
 790                         subspecies = getOrCreateTaxonForName(taxonName, state);
 791                         if (preferredFlag) {
 792                             parent = linkParentChildNode(species, subspecies, classification, state);
 793                         }
 794                     }
 795                 }
 796                 if (preferredFlag && parent!=taxon) {
 797                     linkParentChildNode(parent, taxon, classification, state);
 798                 }
 799             }
 800
 801             /**
 802              * Link a parent to a child and save it in the current classification
 803              * @param parent: the higher Taxon
 804              * @param child : the lower (or current) Taxon
 805              * return the Taxon from the new created Node
 806              * @param classification
 807              * @param state
 808              */
 809             protected Taxon linkParentChildNode(Taxon parent, Taxon child, Classification classification, STATE state) {
 810                 TaxonNode node =null;
 811                 if (parent != null) {
 812                     parent = (Taxon) getTaxonService().find(parent.getUuid());
 813                     child = (Taxon) getTaxonService().find(child.getUuid());
 814                     //here we do not have to check if the taxon nodes already exists
 815                     //this is done by classification.addParentChild()
 816                     //do not add child node if it already exists
 817                     if(hasTaxonNodeInClassification(child, classification)){
 818                         return child;
 819                     }
 820                     else{
 821                         node = classification.addParentChild(parent, child, state.getRef(), "");
 822                         save(classification, state);
 823                     }
 824                 }
 825                 else {
 826                     child = (Taxon) getTaxonService().find(child.getUuid());
 827                     //do not add child node if it already exists
 828                     if(hasTaxonNodeInClassification(child, classification)){
 829                         return child;
 830                     }
 831                     else{
 832                         node = classification.addChildTaxon(child, state.getRef(), null);
 833                         save(classification, state);
 834                     }
 835                 }
 836                 if(node!=null){
 837                     state.getReport().addTaxonNode(node);
 838                     return node.getTaxon();
 839                 }
 840                 String message = "Could not create taxon node for " +child;
 841                 state.getReport().addInfoMessage(message);
 842                 logger.warn(message);
 843                 return null;
 844             }
 845
 846             protected Taxon getOrCreateTaxonForName(TaxonNameBase<?, ?> taxonNameBase, STATE state){
 847
 848                 Set<Taxon> acceptedTaxa = taxonNameBase.getTaxa();
 849                 if(acceptedTaxa.size()>0){
 850                     Taxon firstAcceptedTaxon = acceptedTaxa.iterator().next();
 851                     if(acceptedTaxa.size()>1){
 852                         String message = "More than one accepted taxon was found for taxon name: "
 853                                 + taxonNameBase.getTitleCache() + "!\n" + firstAcceptedTaxon + "was chosen for "+state.getDerivedUnitBase();
 854                         state.getReport().addInfoMessage(message);
 855                         logger.warn(message);
 856                     }
 857                     else{
 858                         return firstAcceptedTaxon;
 859                     }
 860                 }
 861                 else{
 862                     Set<TaxonBase> taxonAndSynonyms = taxonNameBase.getTaxonBases();
 863                     for (TaxonBase taxonBase : taxonAndSynonyms) {
 864                         if(taxonBase.isInstanceOf(Synonym.class)){
 865                             Synonym synonym = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
 866                             Set<Taxon> acceptedTaxaOfSynonym = synonym.getAcceptedTaxa();
 867                             if(acceptedTaxaOfSynonym.size()!=1){
 868                                 String message = "No accepted taxa could be found for taxon name: "
 869                                         + taxonNameBase.getTitleCache()
 870                                         + "!\nEither it is a pro parte synonym or has no accepted taxa";
 871                                 state.getReport().addInfoMessage(message);
 872                                 logger.warn(message);
 873                             }
 874                             else{
 875                                 return acceptedTaxaOfSynonym.iterator().next();
 876                             }
 877                         }
 878                     }
 879                 }
 880                 Taxon taxon = Taxon.NewInstance(taxonNameBase, state.getRef());
 881                 save(taxon, state);
 882                 state.getReport().addTaxon(taxon);
 883                 logger.info("Created new taxon "+ taxon);
 884                 return taxon;
 885             }
 886
 887             private boolean hasTaxonNodeInClassification(Taxon taxon, Classification classification){
 888                 if(taxon.getTaxonNodes()!=null){
 889                     for (TaxonNode node : taxon.getTaxonNodes()){
 890                         if(node.getClassification().equals(classification)){
 891                             return true;
 892                         }
 893                     }
 894                 }
 895                 return false;
 896             }
 897
 898             /**
 899              * HandleIdentifications : get the scientific names present in the ABCD
 900              * document and store link them with the observation/specimen data
 901              * @param state: the current ABCD import state
 902              * @param derivedUnitFacade : the current derivedunitfacade
 903              */
 904             protected void handleIdentifications(STATE state, DerivedUnitFacade derivedUnitFacade) {
 905                SpecimenImportConfiguratorBase config = state.getConfig();
 906
 907
 908                 String scientificName = "";
 909                 boolean preferredFlag = false;
 910
 911                 if (state.getDataHolder().getNomenclatureCode() == ""){
 912                     if (config.getNomenclaturalCode() != null){
 913                         if (config.getNomenclaturalCode() != null){
 914                             state.getDataHolder().setNomenclatureCode(config.getNomenclaturalCode().toString());
 915                         }
 916                     }
 917                 }
 918
 919                 for (int i = 0; i < state.getDataHolder().getIdentificationList().size(); i++) {
 920                     Identification identification = state.getDataHolder().getIdentificationList().get(i);
 921                     scientificName = identification.getScientificName().replaceAll(" et ", " & ");
 922
 923                     String preferred = identification.getPreferred();
 924                     if (preferred.equals("1") || preferred.toLowerCase().indexOf("true") != -1 || state.getDataHolder().getIdentificationList().size()==1) {
 925                         preferredFlag = true;
 926                     }
 927                     else {
 928                         preferredFlag = false;
 929                     }
 930                     if (identification.getCode() != null){
 931                     if (identification.getCode().indexOf(':') != -1) {
 932                         state.getDataHolder().setNomenclatureCode(identification.getCode().split(COLON)[1]);
 933                     }
 934                     else{
 935                         state.getDataHolder().setNomenclatureCode(identification.getCode());
 936                     }
 937                     }
 938                     TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(scientificName, null, preferredFlag, state, i);
 939                     Taxon taxon = getOrCreateTaxonForName(taxonName, state);
 940                     addTaxonNode(taxon, state,preferredFlag);
 941                     linkDeterminationEvent(state, taxon, preferredFlag, derivedUnitFacade);
 942                 }
 943             }
 944
 945             /**
 946              * @param taxon : a taxon to add as a node
 947              * @param state : the ABCD import state
 948              */
 949             protected void addTaxonNode(Taxon taxon, STATE state, boolean preferredFlag) {
 950                 SpecimenImportConfiguratorBase config = state.getConfig();
 951                 logger.info("link taxon to a taxonNode "+taxon.getTitleCache());
 952                 //only add nodes if not already existing in current classification or default classification
 953
 954                 //check if node exists in current classification
 955                 //NOTE: we cannot use hasTaxonNodeInClassification() here because we are first creating it here
 956                 if (!existsInClassification(taxon,state.getClassification(), state)){
 957                     if(config.isMoveNewTaxaToDefaultClassification()){
 958                         //check if node exists in default classification
 959                         if (!existsInClassification(taxon, state.getDefaultClassification(), state)){
 960                             addParentTaxon(taxon, state, preferredFlag, state.getDefaultClassification());
 961                         }
 962                     }
 963                     else {
 964                         //add non-existing taxon to current classification
 965                         addParentTaxon(taxon, state, preferredFlag, state.getClassification());
 966                     }
 967                 }
 968             }
 969
 970
 971             private boolean existsInClassification(Taxon taxon, Classification classification, STATE state){
 972                 boolean exist = false;
 973                 ICdmApplicationConfiguration cdmAppController = state.getConfig().getCdmAppController();
 974             if(cdmAppController==null){
 975                 cdmAppController = this;
 976             }
 977             if (classification != null){
 978                 List<UuidAndTitleCache<TaxonNode>> uuidAndTitleCacheOfAllTaxa = cdmAppController.getClassificationService().getTaxonNodeUuidAndTitleCacheOfAcceptedTaxaByClassification(classification.getUuid());
 979                 if (uuidAndTitleCacheOfAllTaxa != null){
 980                         for (UuidAndTitleCache p : uuidAndTitleCacheOfAllTaxa){
 981                             try{
 982                                 if(p.getTitleCache().equals(taxon.getTitleCache())) {
 983                                     exist = true;
 984                                 }
 985                             }
 986                             catch(Exception e){
 987                                 logger.warn("TaxonNode doesn't seem to have a taxon");
 988                             }
 989                         }
 990                 }
 991             }
 992                 return exist;
 993             }
 994
 995             /**
 996              * join DeterminationEvent to the Taxon Object
 997              * @param state : the ABCD import state
 998              * @param taxon: the current Taxon
 999              * @param preferredFlag :if the current name is preferred
1000              * @param derivedFacade : the derived Unit Facade
1001              */
1002             @SuppressWarnings("rawtypes")
1003         protected void linkDeterminationEvent(STATE state, Taxon taxon, boolean preferredFlag,  DerivedUnitFacade derivedFacade) {
1004                 SpecimenImportConfiguratorBase config = state.getConfig();
1005                 if(DEBUG){
1006                     logger.info("start linkdetermination with taxon:" + taxon.getUuid()+", "+taxon);
1007                 }
1008
1009                 DeterminationEvent determinationEvent = DeterminationEvent.NewInstance();
1010                 determinationEvent.setTaxonName(taxon.getName());
1011                 determinationEvent.setPreferredFlag(preferredFlag);
1012
1013                 determinationEvent.setIdentifiedUnit(state.getDerivedUnitBase());
1014                 state.getDerivedUnitBase().addDetermination(determinationEvent);
1015
1016                 if(DEBUG){
1017                     logger.info("NB TYPES INFO: "+ state.getDataHolder().getStatusList().size());
1018                 }
1019                 for (SpecimenTypeDesignationStatus specimenTypeDesignationstatus : state.getDataHolder().getStatusList()) {
1020                     if (specimenTypeDesignationstatus != null) {
1021                         if(DEBUG){
1022                             logger.info("specimenTypeDesignationstatus :"+ specimenTypeDesignationstatus);
1023                         }
1024
1025                         ICdmApplicationConfiguration cdmAppController = config.getCdmAppController();
1026                         if(cdmAppController == null){
1027                             cdmAppController = this;
1028                         }
1029                         specimenTypeDesignationstatus = (SpecimenTypeDesignationStatus) cdmAppController.getTermService().find(specimenTypeDesignationstatus.getUuid());
1030                         //Designation
1031                         TaxonNameBase<?,?> name = taxon.getName();
1032                         SpecimenTypeDesignation designation = SpecimenTypeDesignation.NewInstance();
1033
1034                         designation.setTypeStatus(specimenTypeDesignationstatus);
1035                         designation.setTypeSpecimen(state.getDerivedUnitBase());
1036                         name.addTypeDesignation(designation, true);
1037                     }
1038                 }
1039
1040                 for (String[] fullReference : state.getDataHolder().getReferenceList()) {
1041
1042
1043                     String strReference=fullReference[0];
1044                     String citationDetail = fullReference[1];
1045                     String citationURL = fullReference[2];
1046                     List<Reference> references = getReferenceService().listByTitle(Reference.class, "strReference", MatchMode.EXACT, null, null, null, null, null);
1047
1048                     if (!references.isEmpty()){
1049                         Reference reference = null;
1050                         for (Reference refe: references) {
1051                             if (refe.getTitleCache().equalsIgnoreCase(strReference)) {
1052                                 reference =refe;
1053                                 break;
1054                             }
1055                         }
1056                         if (reference ==null){
1057                             reference = ReferenceFactory.newGeneric();
1058                             reference.setTitleCache(strReference, true);
1059                             save(reference, state);
1060                         }
1061                         determinationEvent.addReference(reference);
1062                     }
1063                 }
1064                 save(state.getDerivedUnitBase(), state);
1065
1066                 if (config.isAddIndividualsAssociationsSuchAsSpecimenAndObservations() && preferredFlag) {
1067                     //do not add IndividualsAssociation to non-preferred taxa
1068                     if(DEBUG){
1069                         logger.info("isDoCreateIndividualsAssociations");
1070                     }
1071
1072                     makeIndividualsAssociation(state, taxon, determinationEvent);
1073
1074                     save(state.getDerivedUnitBase(), state);
1075                 }
1076             }
1077
1078             /**
1079              * create and link each association (specimen, observation..) to the accepted taxon
1080              * @param state : the ABCD import state
1081              * @param taxon: the current Taxon
1082              * @param determinationEvent:the determinationevent
1083              */
1084             protected void makeIndividualsAssociation(STATE state, Taxon taxon, DeterminationEvent determinationEvent) {
1085                 SpecimenImportConfiguratorBase config = state.getConfig();
1086                 SpecimenUserInteraction sui = config.getSpecimenUserInteraction();
1087
1088                 if (DEBUG) {
1089                     logger.info("MAKE INDIVIDUALS ASSOCIATION");
1090                 }
1091
1092                 TaxonDescription taxonDescription = null;
1093                 Set<TaxonDescription> descriptions= taxon.getDescriptions();
1094 //              if (((Abcd206ImportConfigurator) state.getConfig()).isInteractWithUser()){
1095 //                  if(!state.isDescriptionGroupSet()){
1096 //                      taxonDescription = sui.askForDescriptionGroup(descriptions);
1097 //                      state.setDescriptionGroup(taxonDescription);
1098 //                      state.setDescriptionGroupSet(true);
1099 //                  }else{
1100 //                      taxonDescription=state.getDescriptionGroup();
1101 //                  }
1102 //              } else {
1103                     for (TaxonDescription description : descriptions){
1104                         Set<IdentifiableSource> sources =  new HashSet<>();
1105                         sources.addAll(description.getTaxon().getSources());
1106                         sources.addAll(description.getSources());
1107                         for (IdentifiableSource source:sources){
1108                             if(state.getRef().equals(source.getCitation())) {
1109                                 taxonDescription = description;
1110                             }
1111                         }
1112                     }
1113               //  }
1114                 if (taxonDescription == null){
1115                     taxonDescription = TaxonDescription.NewInstance(taxon, false);
1116                     if(sourceNotLinkedToElement(taxonDescription,state.getRef(),null)) {
1117                         taxonDescription.addSource(OriginalSourceType.Import, null, null, state.getRef(), null);
1118                     }
1119                     state.setDescriptionGroup(taxonDescription);
1120                     taxon.addDescription(taxonDescription);
1121                 }
1122
1123                 //PREPARE REFERENCE QUESTIONS
1124
1125                 Map<String,OriginalSourceBase<?>> sourceMap = new HashMap<String, OriginalSourceBase<?>>();
1126
1127                 List<IdentifiableSource> issTmp = new ArrayList<>();//getCommonService().list(IdentifiableSource.class, null, null, null, null);
1128                 List<DescriptionElementSource> issTmp2 = new ArrayList<>();//getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1129
1130                 Set<OriginalSourceBase> osbSet = new HashSet<OriginalSourceBase>();
1131                 if(issTmp2!=null) {
1132                     osbSet.addAll(issTmp2);
1133                 }
1134                 if(issTmp!=null) {
1135                     osbSet.addAll(issTmp);
1136                 }
1137
1138
1139                 addToSourceMap(sourceMap, osbSet);
1140
1141 //              if (((Abcd206ImportConfigurator) state.getConfig()).isInteractWithUser()){
1142 //                  List<OriginalSourceBase<?>> res = null;
1143 //                  if(!state.isDescriptionSourcesSet()){
1144 //                      res = sui.askForSource(sourceMap, "the description group ("+taxon+")",
1145 //                              "The current reference is "+state.getRef().getTitleCache(),getReferenceService(), state.getDataHolder().docSources);
1146 //                      state.setDescriptionRefs(res);
1147 //                      state.setDescriptionSourcesSet(true);
1148 //                  }
1149 //                  else{
1150 //                      res=state.getDescriptionRefs();
1151 //                  }
1152 //                  if(res !=null) {
1153 //                      for (OriginalSourceBase<?> sour:res){
1154 //                          if(sour.isInstanceOf(IdentifiableSource.class)){
1155 //                              try {
1156 //                                  if(sourceNotLinkedToElement(taxonDescription,sour)) {
1157 //                                      taxonDescription.addSource((IdentifiableSource)sour.clone());
1158 //                                  }
1159 //                              } catch (CloneNotSupportedException e) {
1160 //                                  logger.warn("no cloning?");
1161 //                              }
1162 //                          }else{
1163 //                              if(sourceNotLinkedToElement(taxonDescription,sour)) {
1164 //                                  taxonDescription.addSource(OriginalSourceType.Import,null, null, sour.getCitation(),sour.getCitationMicroReference());
1165 //                              }
1166 //                          }
1167 //                      }
1168 //                  }
1169 //              }
1170 //              else {
1171                     if(sourceNotLinkedToElement(taxonDescription,state.getRef(),null)) {
1172                         taxonDescription.addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1173                     }
1174 //              }
1175                 state.setDescriptionGroup(taxonDescription);
1176
1177                 IndividualsAssociation indAssociation = IndividualsAssociation.NewInstance();
1178                 Feature feature = makeFeature(state.getDerivedUnitBase());
1179                 indAssociation.setAssociatedSpecimenOrObservation(state.getDerivedUnitBase());
1180                 indAssociation.setFeature(feature);
1181 //
1182 //              if (((Abcd206ImportConfigurator) state.getConfig()).isInteractWithUser()){
1183 //                  sourceMap = new HashMap<String, OriginalSourceBase<?>>();
1184 //
1185 //                  issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
1186 //                  issTmp2 = getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1187 //
1188 //                  osbSet = new HashSet<OriginalSourceBase>();
1189 //                  if(issTmp2!=null) {
1190 //                      osbSet.addAll(issTmp2);
1191 //                  }
1192 //                  if(issTmp!=null) {
1193 //                      osbSet.addAll(issTmp);
1194 //                  }
1195 //
1196 //
1197 //                  addToSourceMap(sourceMap, osbSet);
1198 //
1199 //                  List<OriginalSourceBase<?>> sources =null;
1200 //                  if(!state.isAssociationSourcesSet()) {
1201 //                      sources = sui.askForSource(sourceMap,  "descriptive element (association) ",taxon.toString(),
1202 //                              getReferenceService(),state.getDataHolder().getDocSources());
1203 //                      state.setAssociationRefs(sources);
1204 //                      state.setAssociationSourcesSet(true);
1205 //                  }
1206 //                  else{
1207 //                      sources=state.getAssociationRefs();
1208 //                  }
1209 //                  if(sources !=null) {
1210 //                      for (OriginalSourceBase<?> source: sources) {
1211 //                          if(source !=null) {
1212 //                              if(source.isInstanceOf(DescriptionElementSource.class)){
1213 //                                  try {
1214 //                                      if(sourceNotLinkedToElement(indAssociation,source)) {
1215 //                                          indAssociation.addSource((DescriptionElementSource)source.clone());
1216 //                                      }
1217 //                                  } catch (CloneNotSupportedException e) {
1218 //                                      logger.warn("clone forbidden?");
1219 //                                  }
1220 //                              }else{
1221 //                                  if(sourceNotLinkedToElement(indAssociation,source)) {
1222 //                                      indAssociation.addSource(OriginalSourceType.Import,null, null, source.getCitation(),source.getCitationMicroReference());
1223 //                                  }
1224 //                                  try {
1225 //                                      if(sourceNotLinkedToElement(state.getDerivedUnitBase(), source)) {
1226 //                                          state.getDerivedUnitBase().addSource((IdentifiableSource) source.clone());
1227 //                                      }
1228 //                                  } catch (CloneNotSupportedException e) {
1229 //                                      // TODO Auto-generated catch block
1230 //                                      e.printStackTrace();
1231 //                                  }
1232 //                              }
1233 //
1234 //                          }
1235 //                      }
1236 //                  }
1237 //              }else {
1238                     if(sourceNotLinkedToElement(indAssociation,state.getRef(),null)) {
1239                         indAssociation.addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1240                     }
1241                     if(sourceNotLinkedToElement(state.getDerivedUnitBase(), state.getRef(),null)) {
1242                         state.getDerivedUnitBase().addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1243                     }
1244                     for (Reference citation : determinationEvent.getReferences()) {
1245                         if(sourceNotLinkedToElement(indAssociation,citation,null))
1246                         {
1247                             indAssociation.addSource(DescriptionElementSource.NewInstance(OriginalSourceType.Import, null, null, citation, null));
1248                         }
1249                         if(sourceNotLinkedToElement(state.getDerivedUnitBase(), state.getRef(),null)) {
1250                             state.getDerivedUnitBase().addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1251                         }
1252                     }
1253            //     }
1254
1255                 taxonDescription.addElement(indAssociation);
1256
1257                 save(taxonDescription, state);
1258                 save(taxon, state);
1259                 state.getReport().addDerivate(state.getDerivedUnitBase(), config);
1260                 state.getReport().addIndividualAssociation(taxon, state.getDataHolder().getUnitID(), state.getDerivedUnitBase());
1261             }
1262
1263             /**
1264              * @param derivedUnitBase2
1265              * @param ref2
1266              * @param object
1267              * @return
1268              */
1269             private boolean sourceNotLinkedToElement(DerivedUnit derivedUnitBase2, Reference b, String d) {
1270                 Set<IdentifiableSource> linkedSources = derivedUnitBase2.getSources();
1271                 for (IdentifiableSource is:linkedSources){
1272                     Reference a = is.getCitation();
1273                     String c = is.getCitationMicroReference();
1274
1275                     boolean refMatch=false;
1276                     boolean microMatch=false;
1277
1278                     try{
1279                         if (a==null && b==null) {
1280                             refMatch=true;
1281                         }
1282                         if (a!=null && b!=null) {
1283                             if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1284                                 refMatch=true;
1285                             }
1286                         }
1287                     }catch(Exception e){}
1288
1289
1290                     try{
1291                         if (c==null && d==null) {
1292                             microMatch=true;
1293                         }
1294                         if(c!=null && d!=null) {
1295                             if(c.equalsIgnoreCase(d)) {
1296                                 microMatch=true;
1297                             }
1298                         }
1299                     }
1300                     catch(Exception e){}
1301
1302                     if (microMatch && refMatch) {
1303                         return false;
1304                     }
1305
1306
1307                 }
1308                 return true;
1309             }
1310
1311             /**
1312              * @param specimen
1313              * @param source
1314              * @return
1315              */
1316             private boolean sourceNotLinkedToElement(SpecimenOrObservationBase<?> specimen, OriginalSourceBase<?> source) {
1317                 Set<IdentifiableSource> linkedSources = specimen.getSources();
1318                 for (IdentifiableSource is:linkedSources){
1319                     Reference a = is.getCitation();
1320                     Reference b = source.getCitation();
1321                     String c = is.getCitationMicroReference();
1322                     String d = source.getCitationMicroReference();
1323
1324                     boolean refMatch=false;
1325                     boolean microMatch=false;
1326
1327                     try{
1328                         if (a==null && b==null) {
1329                             refMatch=true;
1330                         }
1331                         if (a!=null && b!=null) {
1332                             if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1333                                 refMatch=true;
1334                             }
1335                         }
1336                     }catch(Exception e){}
1337
1338
1339                     try{
1340                         if (c==null && d==null) {
1341                             microMatch=true;
1342                         }
1343                         if(c!=null && d!=null) {
1344                             if(c.equalsIgnoreCase(d)) {
1345                                 microMatch=true;
1346                             }
1347                         }
1348                     }
1349                     catch(Exception e){}
1350
1351                     if (microMatch && refMatch) {
1352                         return false;
1353                     }
1354
1355
1356                 }
1357                 return true;
1358             }
1359
1360             /**
1361              * @param indAssociation
1362              * @param ref2
1363              * @param object
1364              * @return
1365              */
1366             private boolean sourceNotLinkedToElement(IndividualsAssociation indAssociation, Reference a, String d) {
1367                 Set<DescriptionElementSource> linkedSources = indAssociation.getSources();
1368                 for (DescriptionElementSource is:linkedSources){
1369                     Reference b = is.getCitation();
1370                     String c = is.getCitationMicroReference();
1371
1372                     boolean refMatch=false;
1373                     boolean microMatch=false;
1374
1375                     try{
1376                         if (a==null && b==null) {
1377                             refMatch=true;
1378                         }
1379                         if (a!=null && b!=null) {
1380                             if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1381                                 refMatch=true;
1382                             }
1383                         }
1384                     }catch(Exception e){}
1385
1386
1387                     try{
1388                         if (c==null && d==null) {
1389                             microMatch=true;
1390                         }
1391                         if(c!=null && d!=null) {
1392                             if(c.equalsIgnoreCase(d)) {
1393                                 microMatch=true;
1394                             }
1395                         }
1396                     }
1397                     catch(Exception e){}
1398
1399                     if (microMatch && refMatch) {
1400                         return false;
1401                     }
1402                 }
1403                 return true;
1404             }
1405
1406             /**
1407              * @param taxonDescription
1408              * @param ref2
1409              * @param object
1410              * @return
1411              */
1412             private boolean sourceNotLinkedToElement(TaxonDescription taxonDescription, Reference a, String d) {
1413                 Set<IdentifiableSource> linkedSources = taxonDescription.getSources();
1414                 for (IdentifiableSource is:linkedSources){
1415                     Reference b = is.getCitation();
1416                     String c = is.getCitationMicroReference();
1417
1418                     boolean refMatch=false;
1419                     boolean microMatch=false;
1420
1421                     try{
1422                         if (a==null && b==null) {
1423                             refMatch=true;
1424                         }
1425                         if (a!=null && b!=null) {
1426                             if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1427                                 refMatch=true;
1428                             }
1429                         }
1430                     }catch(Exception e){}
1431
1432
1433                     try{
1434                         if (c==null && d==null) {
1435                             microMatch=true;
1436                         }
1437                         if(c!=null && d!=null) {
1438                             if(c.equalsIgnoreCase(d)) {
1439                                 microMatch=true;
1440                             }
1441                         }
1442                     }
1443                     catch(Exception e){}
1444
1445                     if (microMatch && refMatch) {
1446                         return false;
1447                     }
1448                 }
1449                 return true;
1450             }
1451
1452             /**
1453              * @param indAssociation
1454              * @param source
1455              * @return
1456              */
1457             private boolean sourceNotLinkedToElement(IndividualsAssociation indAssociation, OriginalSourceBase<?> source) {
1458                 Set<DescriptionElementSource> linkedSources = indAssociation.getSources();
1459                 for (DescriptionElementSource is:linkedSources){
1460                     Reference a = is.getCitation();
1461                     Reference b = source.getCitation();
1462                     String c = is.getCitationMicroReference();
1463                     String d = source.getCitationMicroReference();
1464
1465                     boolean refMatch=false;
1466                     boolean microMatch=false;
1467
1468                     try{
1469                         if (a==null && b==null) {
1470                             refMatch=true;
1471                         }
1472                         if (a!=null && b!=null) {
1473                             if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1474                                 refMatch=true;
1475                             }
1476                         }
1477                     }catch(Exception e){}
1478
1479
1480                     try{
1481                         if (c==null && d==null) {
1482                             microMatch=true;
1483                         }
1484                         if(c!=null && d!=null) {
1485                             if(c.equalsIgnoreCase(d)) {
1486                                 microMatch=true;
1487                             }
1488                         }
1489                     }
1490                     catch(Exception e){}
1491
1492                     if (microMatch && refMatch) {
1493                         return false;
1494                     }
1495                 }
1496                 return true;
1497             }
1498
1499             /**
1500              * @param taxonDescription
1501              * @param sour
1502              * @return
1503              */
1504             private boolean sourceNotLinkedToElement(TaxonDescription taxonDescription, OriginalSourceBase<?> sour) {
1505                 Set<IdentifiableSource> linkedSources = taxonDescription.getSources();
1506                 for (IdentifiableSource is:linkedSources){
1507                     Reference a = is.getCitation();
1508                     Reference b = sour.getCitation();
1509                     String c = is.getCitationMicroReference();
1510                     String d = sour.getCitationMicroReference();
1511
1512                     boolean refMatch=false;
1513                     boolean microMatch=false;
1514
1515                     try{
1516                         if (a==null && b==null) {
1517                             refMatch=true;
1518                         }
1519                         if (a!=null && b!=null) {
1520                             if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1521                                 refMatch=true;
1522                             }
1523                         }
1524                     }catch(Exception e){}
1525
1526
1527                     try{
1528                         if (c==null && d==null) {
1529                             microMatch=true;
1530                         }
1531                         if(c!=null && d!=null) {
1532                             if(c.equalsIgnoreCase(d)) {
1533                                 microMatch=true;
1534                             }
1535                         }
1536                     }
1537                     catch(Exception e){}
1538
1539                     if (microMatch && refMatch) {
1540                         return false;
1541                     }
1542
1543
1544                 }
1545                 return true;
1546             }
1547
1548             /**
1549              * look for the Feature object (FieldObs, Specimen,...)
1550              * @param unit : a specimen or obersvation base
1551              * @return the corresponding Feature
1552              */
1553             private Feature makeFeature(SpecimenOrObservationBase<?> unit) {
1554                 SpecimenOrObservationType type = unit.getRecordBasis();
1555
1556
1557
1558                 if (type.isFeatureObservation()){
1559                     return Feature.OBSERVATION();
1560                 }else if (type.isFeatureSpecimen()){
1561                     return Feature.SPECIMEN();
1562                 }else if (type == SpecimenOrObservationType.DerivedUnit){
1563                     return Feature.OBSERVATION();
1564                     //            return getFeature("Specimen or observation");
1565                 }else{
1566                     String message = "Unhandled record basis '%s' for defining individuals association feature type. Use default.";
1567                     logger.warn(String.format(message, type.getMessage()));
1568                     return Feature.OBSERVATION();
1569                     //            return getFeature("Specimen or observation");
1570
1571                 }
1572             }
1573
1574
1575             /**
1576              * @param sourceMap
1577              * @param osbSet
1578              */
1579             protected void addToSourceMap(Map<String, OriginalSourceBase<?>> sourceMap, Set<OriginalSourceBase> osbSet) {
1580                 for( OriginalSourceBase<?> osb:osbSet) {
1581                     if(osb.getCitation()!=null && osb.getCitationMicroReference() !=null  && !osb.getCitationMicroReference().isEmpty()) {
1582                         try{
1583                             sourceMap.put(osb.getCitation().getTitleCache()+ "---"+osb.getCitationMicroReference(),osb);
1584                         }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1585                     } else if(osb.getCitation()!=null){
1586                         try{
1587                             sourceMap.put(osb.getCitation().getTitleCache(),osb);
1588                         }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1589                     }
1590                 }
1591             }
1592
1593 }