cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/abcd206/in/Abcd206Import.java

   1 /**
   2  * Copyright (C) 2007 EDIT
   3  * European Distributed Institute of Taxonomy
   4  * http://www.e-taxonomy.eu
   5  *
   6  * The contents of this file are subject to the Mozilla Public License Version 1.1
   7  * See LICENSE.TXT at the top of this package for the full license terms.
   8  */
   9
  10 package eu.etaxonomy.cdm.io.specimen.abcd206.in;
  11
  12 import java.io.InputStream;
  13 import java.net.MalformedURLException;
  14 import java.net.URI;
  15 import java.util.ArrayList;
  16 import java.util.HashMap;
  17 import java.util.List;
  18 import java.util.Map;
  19 import java.util.Set;
  20 import java.util.UUID;
  21
  22 import org.apache.commons.lang.StringUtils;
  23 import org.apache.log4j.Logger;
  24 import org.springframework.stereotype.Component;
  25 import org.w3c.dom.Element;
  26 import org.w3c.dom.NodeList;
  27
  28 import eu.etaxonomy.cdm.api.application.ICdmRepository;
  29 import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
  30 import eu.etaxonomy.cdm.ext.occurrence.bioCase.BioCaseQueryServiceWrapper;
  31 import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
  32 import eu.etaxonomy.cdm.io.common.ICdmIO;
  33 import eu.etaxonomy.cdm.io.common.MapWrapper;
  34 import eu.etaxonomy.cdm.io.specimen.SpecimenImportBase;
  35 import eu.etaxonomy.cdm.io.specimen.SpecimenUserInteraction;
  36 import eu.etaxonomy.cdm.io.specimen.UnitsGatheringArea;
  37 import eu.etaxonomy.cdm.io.specimen.UnitsGatheringEvent;
  38 import eu.etaxonomy.cdm.io.specimen.abcd206.in.molecular.AbcdDnaParser;
  39 import eu.etaxonomy.cdm.model.agent.Institution;
  40 import eu.etaxonomy.cdm.model.agent.Person;
  41 import eu.etaxonomy.cdm.model.agent.Team;
  42 import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;
  43 import eu.etaxonomy.cdm.model.common.Annotation;
  44 import eu.etaxonomy.cdm.model.common.CdmBase;
  45 import eu.etaxonomy.cdm.model.common.DefinedTerm;
  46 import eu.etaxonomy.cdm.model.common.DefinedTermBase;
  47 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  48 import eu.etaxonomy.cdm.model.common.Language;
  49 import eu.etaxonomy.cdm.model.common.LanguageString;
  50 import eu.etaxonomy.cdm.model.common.OriginalSourceBase;
  51 import eu.etaxonomy.cdm.model.common.OriginalSourceType;
  52 import eu.etaxonomy.cdm.model.location.NamedArea;
  53 import eu.etaxonomy.cdm.model.media.Media;
  54 import eu.etaxonomy.cdm.model.molecular.DnaSample;
  55 import eu.etaxonomy.cdm.model.occurrence.Collection;
  56 import eu.etaxonomy.cdm.model.occurrence.DerivationEvent;
  57 import eu.etaxonomy.cdm.model.occurrence.DerivationEventType;
  58 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  59 import eu.etaxonomy.cdm.model.occurrence.FieldUnit;
  60 import eu.etaxonomy.cdm.model.occurrence.GatheringEvent;
  61 import eu.etaxonomy.cdm.model.occurrence.MediaSpecimen;
  62 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
  63 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationType;
  64 import eu.etaxonomy.cdm.model.reference.Reference;
  65 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  66 import eu.etaxonomy.cdm.model.taxon.Classification;
  67
  68 /**
  69  * @author p.kelbert
  70  * @author p.plitzner
  71  * @created 20.10.2008
  72  */
  73 @Component
  74 public class Abcd206Import extends SpecimenImportBase<Abcd206ImportConfigurator, Abcd206ImportState> {
  75
  76     private static final long serialVersionUID = 3918095362150986307L;
  77
  78     private static final UUID SPECIMEN_SCAN_TERM = UUID.fromString("acda15be-c0e2-4ea8-8783-b9b0c4ad7f03");
  79
  80     private static final Logger logger = Logger.getLogger(Abcd206Import.class);
  81
  82
  83     public Abcd206Import() {
  84         super();
  85     }
  86
  87
  88     @Override
  89 //    @SuppressWarnings("rawtypes")
  90     public void doInvoke(Abcd206ImportState state) {
  91         Abcd206ImportConfigurator config = state.getConfig();
  92         Map<String, MapWrapper<? extends CdmBase>> stores = state.getStores();
  93         MapWrapper<TeamOrPersonBase<?>> authorStore = (MapWrapper<TeamOrPersonBase<?>>)stores.get(ICdmIO.TEAM_STORE);
  94         state.setPersonStore(authorStore);
  95         MapWrapper<Reference> referenceStore = (MapWrapper<Reference>)stores.get(ICdmIO.REFERENCE_STORE);
  96         URI sourceUri = config.getSourceUri();
  97         try{
  98             state.setTx(startTransaction());
  99             logger.info("INVOKE Specimen Import from ABCD2.06 XML ");
 100             InputStream response = null;
 101             //init cd repository
 102             if(state.getCdmRepository()==null){
 103                 state.setCdmRepository(this);
 104             }
 105             if (config.getOccurenceQuery() != null){
 106                 BioCaseQueryServiceWrapper queryService = new BioCaseQueryServiceWrapper();
 107                 try {
 108
 109                    response = queryService.query(config.getOccurenceQuery(), sourceUri);
 110                    state.setActualAccessPoint(sourceUri);
 111
 112                 }catch(Exception e){
 113                     logger.error("An error during ABCD import");
 114                 }
 115             }
 116             SpecimenUserInteraction sui = state.getConfig().getSpecimenUserInteraction();
 117
 118             //init import reference
 119         //  List<Reference> references = getReferenceService().list(Reference.class, null, null, null, null);
 120          //   List<Reference> references = new ArrayList<Reference>();
 121
 122 //            if (state.getConfig().isInteractWithUser()){
 123 //                Map<String,Reference> refMap = new HashMap<String, Reference>();
 124 //                for (Reference reference : references) {
 125 //                    if (! StringUtils.isBlank(reference.getTitleCache())) {
 126 //                        refMap.put(reference.getTitleCache(),reference);
 127 //                    }
 128 //                }
 129 //                state.setRef(sui.askForReference(refMap));
 130 //
 131 //                if (state.getRef() == null){
 132 //                    String cla = sui.createNewReference();
 133 //                    if (refMap.get(cla)!= null) {
 134 //                        state.setRef(refMap.get(cla));
 135 //                    } else {
 136 //                        state.setRef(ReferenceFactory.newGeneric());
 137 //                        state.getRef().setTitle(cla);
 138 //                    }
 139 //                }
 140 //                else{
 141 //                    state.setRef(getReferenceService().find(state.getRef().getUuid()));
 142 //                }
 143 //            }else{
 144                 if (state.getRef()==null){
 145                     String name = NB(state.getConfig().getSourceReferenceTitle());
 146                     for (Reference reference : referenceStore.getAllValues()) {
 147                         if (! StringUtils.isBlank(reference.getTitleCache())) {
 148                             if (reference.getTitleCache().equalsIgnoreCase(name)) {
 149                                 state.setRef(reference);
 150                             }
 151                         }
 152                     }
 153                     if (state.getRef() == null){
 154                         if (state.getConfig().getSourceReference() != null){
 155                             state.setRef(state.getConfig().getSourceReference());
 156                         }else{
 157                             state.setRef(ReferenceFactory.newGeneric());
 158
 159                             if (state.getConfig().getSourceReferenceTitle() != null){
 160                                 state.getRef().setTitle(state.getConfig().getSourceReferenceTitle());
 161                             } else{
 162                                 state.getRef().setTitle("ABCD Import Source Reference");
 163                             }
 164                         }
 165
 166                     }
 167                 }
 168             //}
 169
 170             save(state.getRef(), state);
 171             state.getConfig().setSourceReference(state.getRef());
 172
 173             if(state.getConfig().getClassificationUuid()!=null){
 174                 //load classification from config if it exists
 175                 state.setClassification(getClassificationService().load(state.getConfig().getClassificationUuid()));
 176             }
 177             if(state.getClassification()==null){//no existing classification was set in config
 178                 List<Classification> classificationList = getClassificationService().list(Classification.class, null, null, null, null);
 179                 //get classification via user interaction
 180                 if (state.getConfig().isUseClassification() && state.getConfig().isInteractWithUser()){
 181                     Map<String,Classification> classMap = new HashMap<>();
 182                     for (Classification tree : classificationList) {
 183                         if (! StringUtils.isBlank(tree.getTitleCache())) {
 184                             classMap.put(tree.getTitleCache(),tree);
 185                         }
 186                     }
 187                     state.setClassification(sui.askForClassification(classMap));
 188                     if (state.getClassification() == null){
 189                         String cla = sui.createNewClassification();
 190                         if (classMap.get(cla)!= null) {
 191                             state.setClassification(classMap.get(cla));
 192                         } else {
 193                             state.setClassification(Classification.NewInstance(cla, state.getRef(), Language.DEFAULT()));
 194                         }
 195                     }
 196                     save(state.getClassification(), state);
 197                 }
 198                 // use default classification as the classification to import into
 199                 if (state.getClassification() == null) {
 200                     String name = NB(state.getConfig().getClassificationName());
 201                     for (Classification classif : classificationList){
 202                         if (classif.getTitleCache() != null && classif.getTitleCache().equalsIgnoreCase(name)) {
 203                             state.setClassification(classif);
 204                         }
 205                     }
 206                     if (state.getClassification() == null){
 207                         state.setClassification(Classification.NewInstance(name, state.getRef(), Language.DEFAULT()));
 208                         //we do not need a default classification when creating an empty new one
 209                         state.setDefaultClassification(state.getClassification());
 210                         save(state.getDefaultClassification(false), state);
 211                     }
 212                     save(state.getClassification(), state);
 213                 }
 214             }
 215
 216             if (response == null){
 217                 response =state.getConfig().getSource();
 218             }
 219             UnitAssociationWrapper unitAssociationWrapper = AbcdParseUtility.parseUnitsNodeList(response, state.getReport());
 220             NodeList unitsList = unitAssociationWrapper.getAssociatedUnits();
 221             state.setPrefix(unitAssociationWrapper.getPrefix());
 222
 223             if (unitsList != null) {
 224                 String message = "nb units to insert: " + unitsList.getLength();
 225                 logger.info(message);
 226                 state.getConfig().getProgressMonitor().beginTask("Importing ABCD file", unitsList.getLength() + 3);
 227                 updateProgress(state, message);
 228
 229                 state.setDataHolder(new Abcd206DataHolder());
 230                 state.getDataHolder().reset();
 231
 232                 Abcd206XMLFieldGetter abcdFieldGetter = new Abcd206XMLFieldGetter(state.getDataHolder(), state.getPrefix());
 233
 234                 prepareCollectors(state, unitsList, abcdFieldGetter);
 235
 236
 237                 // save authors
 238                 getAgentService().saveOrUpdate((java.util.Collection)state.getPersonStore().objects());
 239
 240                 commitTransaction(state.getTx());
 241                 state.setTx(startTransaction());
 242                 if (state.getDefaultClassification(false) != null){
 243                     state.setDefaultClassification(getClassificationService().load(state.getDefaultClassification(false).getUuid()));
 244                 }
 245                 if (state.getClassification() != null){
 246                     state.setClassification(getClassificationService().load(state.getClassification().getUuid()));
 247                 }
 248                 state.setAssociationRefs(new ArrayList<>());
 249                 state.setDescriptionRefs(new ArrayList<>());
 250                 state.setDerivedUnitSources(new ArrayList<>());
 251                 for (int i = 0; i < unitsList.getLength(); i++) {
 252                     if(state.getConfig().getProgressMonitor().isCanceled()){
 253                         break;
 254                     }
 255
 256                     state.reset();
 257
 258                     Element item = (Element) unitsList.item(i);
 259                     Abcd206ImportParser.setUnitPropertiesXML( item, abcdFieldGetter, state);
 260                     updateProgress(state, "Importing data for unit "+state.getDataHolder().getUnitID()+" ("+i+"/"+unitsList.getLength()+")");
 261
 262                     //import unit + field unit data
 263                     state.setAssociatedUnitIds(state.getDataHolder().getAssociatedUnitIds());
 264                     this.handleSingleUnit(state, item, true);
 265
 266                 }
 267                 if(state.getConfig().isDeduplicateReferences()){
 268                     getReferenceService().deduplicate(Reference.class, null, null);
 269                 }
 270                 if(state.getConfig().isDeduplicateClassifications()){
 271                     getClassificationService().deduplicate(Classification.class, null, null);
 272                 }
 273             }
 274             commitTransaction(state.getTx());
 275         }
 276         catch(Exception e){
 277             String errorDuringImport = "Exception during import!";
 278             logger.error(errorDuringImport, e);
 279             state.getReport().addException(errorDuringImport, e);
 280         }
 281         finally{
 282             state.getReport().printReport(state.getConfig().getReportUri());
 283         }
 284         return;
 285     }
 286
 287
 288     /**
 289      * @param state
 290      * @param item
 291      */
 292     private void getSiblings(Abcd206ImportState state, Object item, DerivedUnitFacade facade) {
 293         String unitId = facade.getCatalogNumber();
 294
 295         UnitAssociationParser unitParser = new UnitAssociationParser(state.getPrefix(), state.getReport(), state.getCdmRepository());
 296         UnitAssociationWrapper unitAssociationWrapper = null;
 297         for (URI accessPoint: state.getActualAccesPoint()){
 298             unitAssociationWrapper = unitParser.parseSiblings(unitId, accessPoint);
 299             if (unitAssociationWrapper.getAssociatedUnits() != null){
 300                 break;
 301             }
 302         }
 303
 304        DerivedUnit currentUnit = state.getDerivedUnitBase();
 305      //  DerivationEvent currentDerivedFrom = currentUnit.getDerivedFrom();
 306        FieldUnit currentFieldUnit = facade.getFieldUnit(false);
 307        if(unitAssociationWrapper!=null){
 308             NodeList associatedUnits = unitAssociationWrapper.getAssociatedUnits();
 309             if(associatedUnits!=null){
 310                 for(int m=0;m<associatedUnits.getLength();m++){
 311                     if(associatedUnits.item(m) instanceof Element){
 312                         state.reset();
 313                         String associationType = AbcdParseUtility.parseFirstTextContent(((Element) associatedUnits.item(m)).getElementsByTagName(state.getPrefix()+"AssociationType"));
 314
 315                         Abcd206ImportParser.setUnitPropertiesXML((Element) associatedUnits.item(m), new Abcd206XMLFieldGetter(state.getDataHolder(), unitAssociationWrapper.getPrefix()), state);
 316                        // logger.debug("derived unit: " + state.getDerivedUnitBase().toString() + " associated unit: " +state.getDataHolder().getKindOfUnit() + ", " + state.getDataHolder().accessionNumber + ", " + state.getDataHolder().getRecordBasis() + ", " + state.getDataHolder().getUnitID());
 317
 318                         handleSingleUnit(state, associatedUnits.item(m), false);
 319
 320                         DerivedUnit associatedUnit = state.getDerivedUnitBase();
 321                         FieldUnit associatedFieldUnit = null;
 322                         java.util.Collection<FieldUnit> associatedFieldUnits = state.getCdmRepository().getOccurrenceService().getFieldUnits(associatedUnit.getUuid());
 323                         //ignore field unit if associated unit has more than one
 324                         if(associatedFieldUnits.size()>1){
 325                             state.getReport().addInfoMessage(String.format("%s has more than one field unit.", associatedUnit));
 326                         }
 327                         else if(associatedFieldUnits.size()==1){
 328                             associatedFieldUnit = associatedFieldUnits.iterator().next();
 329                         }
 330                         //parent-child relation:
 331                         if(associationType.contains("individual") || associationType.contains("culture") || associationType.contains("sample")){
 332                             DerivationEvent updatedDerivationEvent = DerivationEvent.NewSimpleInstance(currentUnit, associatedUnit, DerivationEventType.ACCESSIONING());
 333                             if(associatedFieldUnit!=null && associatedFieldUnit != currentFieldUnit){
 334                                 associatedFieldUnit.removeDerivationEvent(updatedDerivationEvent);
 335                                 state.getCdmRepository().getOccurrenceService().delete(associatedFieldUnit);
 336                             }
 337                             state.getReport().addDerivate(associatedUnit, currentUnit, state.getConfig());
 338                         }
 339                         save(associatedUnit, state);
 340
 341                     }
 342                 }
 343             }
 344         }
 345         state.reset();
 346         state.setDerivedUnitBase(currentUnit);
 347
 348     }
 349
 350     /**
 351      * Handle a single unit
 352      * @param state
 353      * @param item
 354      */
 355     @Override
 356     public void handleSingleUnit(Abcd206ImportState state, Object itemObject){
 357         handleSingleUnit(state, itemObject, true);
 358     }
 359
 360
 361     @SuppressWarnings("rawtypes")
 362     public void handleSingleUnit(Abcd206ImportState state, Object itemObject, boolean handleAssociatedUnits) {
 363         Element item = (Element) itemObject;
 364
 365         Abcd206ImportConfigurator config = state.getConfig();
 366         if (logger.isDebugEnabled()) {
 367             logger.info("handleSingleUnit "+state.getRef());
 368         }
 369         try {
 370             ICdmRepository cdmAppController = state.getConfig().getCdmAppController();
 371             if(cdmAppController==null){
 372                 cdmAppController = this;
 373             }
 374             //check if unit already exists
 375             DerivedUnitFacade derivedUnitFacade = null;
 376             if(state.getConfig().isIgnoreImportOfExistingSpecimen()){
 377                 SpecimenOrObservationBase<?> existingSpecimen = findExistingSpecimen(state.getDataHolder().getUnitID(), state);
 378                 if(existingSpecimen!=null && existingSpecimen.isInstanceOf(DerivedUnit.class)){
 379                     DerivedUnit derivedUnit = HibernateProxyHelper.deproxy(existingSpecimen, DerivedUnit.class);
 380                     state.setDerivedUnitBase(derivedUnit);
 381                     derivedUnitFacade = DerivedUnitFacade.NewInstance(state.getDerivedUnitBase());
 382                     if (handleAssociatedUnits){
 383                         importAssociatedUnits(state, item, derivedUnitFacade);
 384                     }
 385                     state.getReport().addAlreadyExistingSpecimen(SpecimenImportUtility.getUnitID(derivedUnit, config), derivedUnit);
 386                     return;
 387                 }
 388             }
 389             // TODO: implement overwrite/merge specimen
 390 //            else if(state.getConfig().isOverwriteExistingSpecimens()){
 391 //                Pager<SpecimenOrObservationBase> existingSpecimens = cdmAppController.getOccurrenceService().findByTitle(config);
 392 //                if(!existingSpecimens.getRecords().isEmpty()){
 393 //                    derivedUnitFacade = DerivedUnitFacade.NewInstance(derivedUnit);
 394 //                    derivedUnitBase = derivedUnitFacade.innerDerivedUnit();
 395 //                    fieldUnit = derivedUnitFacade.getFieldUnit(true);
 396 //                }
 397 //            }
 398             //import new specimen
 399
 400             // import DNA unit
 401             if(state.getDataHolder().getKindOfUnit() !=null && state.getDataHolder().getKindOfUnit().equalsIgnoreCase("dna")){
 402                 AbcdDnaParser dnaParser = new AbcdDnaParser(state.getPrefix(), state.getReport(), state.getCdmRepository());
 403                 DnaSample dnaSample = dnaParser.parse(item, state);
 404                 save(dnaSample, state);
 405                 //set dna as derived unit to avoid creating an extra specimen for this dna sample (instead just the field unit will be created)
 406                 state.setDerivedUnitBase(dnaSample);
 407                 derivedUnitFacade = DerivedUnitFacade.NewInstance(state.getDerivedUnitBase());
 408             }
 409             else{
 410                 // create facade
 411                 derivedUnitFacade = getFacade(state);
 412                 state.setDerivedUnitBase(derivedUnitFacade.innerDerivedUnit());
 413
 414             }
 415
 416             /**
 417              * GATHERING EVENT
 418              */
 419
 420             //look for existing fieldUnit
 421
 422             FieldUnit fieldUnit = state.getFieldUnit(state.getDataHolder().getFieldNumber());
 423
 424
 425                 // gathering event
 426                 UnitsGatheringEvent unitsGatheringEvent = new UnitsGatheringEvent(cdmAppController.getTermService(),
 427                         state.getDataHolder().locality, state.getDataHolder().languageIso, state.getDataHolder().longitude,
 428                         state.getDataHolder().latitude, state.getDataHolder().getGatheringElevationText(),
 429                         state.getDataHolder().getGatheringElevationMin(), state.getDataHolder().getGatheringElevationMax(),
 430                         state.getDataHolder().getGatheringElevationUnit(), state.getDataHolder().getGatheringDateText(),
 431                         state.getDataHolder().getGatheringNotes(), state.getDataHolder().getGatheringMethod(), state.getTransformer().getReferenceSystemByKey(
 432                                 state.getDataHolder().getGatheringSpatialDatum()),
 433                          state.getConfig());
 434
 435                 unitsGatheringEvent.setGatheringDepth(state.getDataHolder().getGatheringDepthText(),state.getDataHolder().getGatheringDepthMin(), state.getDataHolder().getGatheringDepthMax(), state.getDataHolder().getGatheringDepthUnit());
 436                 //unitsGatheringEvent.setHeight(heightText, heightMin, heightMax, heightUnit);
 437                 unitsGatheringEvent.setCollector(state.getPersonStore().get(state.getDataHolder().gatheringAgents), config);
 438                 // count
 439                 UnitsGatheringArea unitsGatheringArea = new UnitsGatheringArea();
 440                 //  unitsGatheringArea.setConfig(state.getConfig(),getOccurrenceService(), getTermService());
 441                 unitsGatheringArea.setParams(state.getDataHolder().isocountry, state.getDataHolder().country, (state.getConfig()), cdmAppController.getTermService(), cdmAppController.getOccurrenceService(), cdmAppController.getVocabularyService());
 442
 443                 DefinedTermBase<?> areaCountry =  unitsGatheringArea.getCountry();
 444
 445                 // other areas
 446                 unitsGatheringArea = new UnitsGatheringArea();
 447                 //            unitsGatheringArea.setConfig(state.getConfig(),getOccurrenceService(),getTermService());
 448
 449                 unitsGatheringArea.setAreas(state.getDataHolder().getNamedAreaList(),(state.getConfig()), cdmAppController.getTermService(), cdmAppController.getVocabularyService());
 450
 451                 ArrayList<DefinedTermBase> nas = unitsGatheringArea.getAreas();
 452                 for (DefinedTermBase namedArea : nas) {
 453                     unitsGatheringEvent.addArea(namedArea);
 454                 }
 455
 456                 // copy gathering event to facade
 457                 GatheringEvent gatheringEvent = unitsGatheringEvent.getGatheringEvent();
 458                 if (fieldUnit != null){
 459                     derivedUnitFacade.setFieldUnit(fieldUnit);
 460                 }
 461
 462                 derivedUnitFacade.setLocality(gatheringEvent.getLocality());
 463                 derivedUnitFacade.setExactLocation(gatheringEvent.getExactLocation());
 464                 derivedUnitFacade.setCollector(gatheringEvent.getCollector());
 465                 derivedUnitFacade.setCountry((NamedArea)areaCountry);
 466                 derivedUnitFacade.setAbsoluteElevationText(gatheringEvent.getAbsoluteElevationText());
 467                 derivedUnitFacade.setAbsoluteElevation(gatheringEvent.getAbsoluteElevation());
 468                 derivedUnitFacade.setAbsoluteElevationMax(gatheringEvent.getAbsoluteElevationMax());
 469                 derivedUnitFacade.setDistanceToGroundText(gatheringEvent.getDistanceToGroundText());
 470                 derivedUnitFacade.setDistanceToGroundMax(gatheringEvent.getDistanceToGroundMax());
 471                 derivedUnitFacade.setDistanceToGround(gatheringEvent.getDistanceToGround());
 472                 derivedUnitFacade.setDistanceToWaterSurfaceText(gatheringEvent.getDistanceToWaterSurfaceText());
 473                 derivedUnitFacade.setDistanceToWaterSurfaceMax(gatheringEvent.getDistanceToWaterSurfaceMax());
 474                 derivedUnitFacade.setDistanceToWaterSurface(gatheringEvent.getDistanceToWaterSurface());
 475                 derivedUnitFacade.setGatheringPeriod(gatheringEvent.getTimeperiod());
 476                 derivedUnitFacade.setCollectingMethod(gatheringEvent.getCollectingMethod());
 477
 478                 for(DefinedTermBase<?> area:unitsGatheringArea.getAreas()){
 479                     derivedUnitFacade.addCollectingArea((NamedArea) area);
 480                 }
 481                 //            derivedUnitFacade.addCollectingAreas(unitsGatheringArea.getAreas());
 482                 // TODO exsiccatum
 483
 484                 // add fieldNumber
 485                 derivedUnitFacade.setFieldNumber(NB(state.getDataHolder().getFieldNumber()));
 486                 save(unitsGatheringEvent.getLocality(), state);
 487
 488             // add unitNotes
 489             if (state.getDataHolder().getUnitNotes() != null){
 490                 derivedUnitFacade.addAnnotation(Annotation.NewDefaultLanguageInstance(NB(state.getDataHolder().getUnitNotes())));
 491             }
 492
 493
 494
 495             // //add Multimedia URLs
 496             if (state.getDataHolder().getMultimediaObjects().size() != -1) {
 497                 for (String multimediaObject : state.getDataHolder().getMultimediaObjects().keySet()) {
 498                     Media media;
 499                     try {
 500                         media = getImageMedia(multimediaObject, READ_MEDIA_DATA);
 501                         Map<String, String> attributes = state.getDataHolder().getMultimediaObjects().get(multimediaObject);
 502                         if (attributes.containsKey("Context")){
 503                             LanguageString description = LanguageString.NewInstance(attributes.get("Context"), Language.ENGLISH());
 504                             media.addDescription(description);
 505                         }
 506                         if (attributes.containsKey("Comment")){
 507                             LanguageString description = LanguageString.NewInstance(attributes.get("Comment"), Language.ENGLISH());
 508                             media.addDescription(description);
 509                         }
 510                         if (attributes.containsKey("Creators")){
 511                             String creators = attributes.get("Creators");
 512                             Person artist;
 513                             Team artistTeam;
 514                             String[] artists;
 515                             if (creators != null){
 516                                 if (creators.contains("&")){
 517                                     artists = creators.split("&");
 518                                     artistTeam = new Team();
 519                                     for (String creator:artists){
 520                                         artist = Person.NewTitledInstance(creator);
 521                                         artistTeam.addTeamMember(artist);
 522                                     }
 523                                     media.setArtist(artistTeam);
 524                                 } else{
 525
 526                                     artist = Person.NewTitledInstance(creators);
 527                                     media.setArtist(artist);
 528                                 }
 529                             }
 530
 531
 532
 533                         }
 534
 535                         derivedUnitFacade.addDerivedUnitMedia(media);
 536                         if(state.getConfig().isAddMediaAsMediaSpecimen()){
 537                             //add media also as specimen scan
 538                             MediaSpecimen mediaSpecimen = MediaSpecimen.NewInstance(SpecimenOrObservationType.StillImage);
 539                             mediaSpecimen.setMediaSpecimen(media);
 540                             //do it only once!!
 541                             DefinedTermBase specimenScanTerm = getTermService().load(SPECIMEN_SCAN_TERM);
 542                             if(specimenScanTerm instanceof DefinedTerm){
 543                                 mediaSpecimen.setKindOfUnit((DefinedTerm) specimenScanTerm);
 544                             }
 545                             DerivationEvent derivationEvent = DerivationEvent.NewInstance(DerivationEventType.PREPARATION());
 546                             derivationEvent.addDerivative(mediaSpecimen);
 547                             derivedUnitFacade.innerDerivedUnit().addDerivationEvent(derivationEvent);
 548                         }
 549
 550                     } catch (MalformedURLException e) {
 551                         // TODO Auto-generated catch block
 552                         e.printStackTrace();
 553                     }
 554
 555                 }
 556             }
 557             //multimedia for fieldUnit
 558             if (state.getDataHolder().getGatheringMultimediaObjects().size() != -1) {
 559                 for (String multimediaObject : state.getDataHolder().getGatheringMultimediaObjects().keySet()) {
 560                     Media media;
 561                     try {
 562                         media = getImageMedia(multimediaObject, READ_MEDIA_DATA);
 563                         Map<String, String> attributes = state.getDataHolder().getGatheringMultimediaObjects().get(multimediaObject);
 564                         if (attributes.containsKey("Context")){
 565                             LanguageString description = LanguageString.NewInstance(attributes.get("Context"), Language.ENGLISH());
 566                             media.addDescription(description);
 567                         }
 568                         if (attributes.containsKey("Comment")){
 569                             LanguageString description = LanguageString.NewInstance(attributes.get("Comment"), Language.ENGLISH());
 570                             media.addDescription(description);
 571                         }
 572                         if (attributes.containsKey("Creators")){
 573                             String creators = attributes.get("Creators");
 574                             Person artist;
 575                             Team artistTeam;
 576                             String[] artists;
 577                             if (creators != null){
 578                                 if (creators.contains("&")){
 579                                     artists = creators.split("&");
 580                                     artistTeam = new Team();
 581                                     for (String creator:artists){
 582                                         artist = Person.NewTitledInstance(creator);
 583                                         artistTeam.addTeamMember(artist);
 584                                     }
 585                                     media.setArtist(artistTeam);
 586                                 } else{
 587
 588                                     artist = Person.NewTitledInstance(creators);
 589                                     media.setArtist(artist);
 590                                 }
 591                             }
 592
 593
 594
 595                         }
 596
 597                         derivedUnitFacade.addFieldObjectMedia(media);
 598
 599
 600                     } catch (MalformedURLException e) {
 601                         // TODO Auto-generated catch block
 602                         e.printStackTrace();
 603                     }
 604
 605                 }
 606             }
 607
 608             //                  /*
 609             //                   * merge AND STORE DATA
 610             //                   */
 611             //                  getTermService().saveOrUpdate(areaCountry);// TODO save area sooner
 612             //
 613             //                  for (NamedArea area : otherAreas) {
 614             //                          getTermService().saveOrUpdate(area);// merge it sooner (foreach area)
 615             //                  }
 616 //           save(derivedUnitFacade.getFieldUnit(false), state);
 617            if (derivedUnitFacade.getFieldUnit(false) != null){
 618                state.setFieldUnit(derivedUnitFacade.getFieldUnit(false));
 619            }
 620
 621             // handle collection data
 622             setCollectionData(state, derivedUnitFacade);
 623
 624             //Reference stuff
 625             SpecimenUserInteraction sui = config.getSpecimenUserInteraction();
 626             Map<String,OriginalSourceBase<?>> sourceMap = new HashMap<>();
 627
 628             state.getDataHolder().setDocSources(new ArrayList<>());
 629             for (String[] fullReference : state.getDataHolder().getReferenceList()) {
 630                 String strReference=fullReference[0];
 631                 String citationDetail = fullReference[1];
 632                 String citationURL = fullReference[2];
 633
 634                 if (!citationURL.isEmpty()) {
 635                     citationDetail+=", "+citationURL;
 636                 }
 637
 638                 Reference reference;
 639                 if(strReference.equals(state.getRef().getTitleCache())){
 640                     reference = state.getRef();
 641                 }
 642                 else{
 643                     reference = ReferenceFactory.newGeneric();
 644                     reference.setTitle(strReference);
 645                 }
 646
 647
 648                 save(reference, state);
 649                 IdentifiableSource sour = getIdentifiableSource(reference, citationDetail);
 650                 sour.getCitation().setUri(state.getActualAccessPoint());
 651                 sour.setType(OriginalSourceType.PrimaryTaxonomicSource);
 652                 try{
 653                     if (sour.getCitation() != null){
 654                         if(StringUtils.isNotBlank(sour.getCitationMicroReference())) {
 655                             state.getDataHolder().getDocSources().add(sour.getCitation().getTitleCache()+ "---"+sour.getCitationMicroReference());
 656                         } else {
 657                             state.getDataHolder().getDocSources().add(sour.getCitation().getTitleCache());
 658                         }
 659                     }
 660                 }catch(Exception e){
 661                     logger.warn("oups");
 662                 }
 663                 derivedUnitFacade.addSource(sour);
 664
 665             }
 666 //            List<IdentifiableSource> issTmp = new ArrayList<IdentifiableSource>();//getCommonService().list(IdentifiableSource.class, null, null, null, null);
 667 //            List<DescriptionElementSource> issTmp2 = new ArrayList<DescriptionElementSource>();//getCommonService().list(DescriptionElementSource.class, null, null, null, null);
 668 //
 669 //            Set<OriginalSourceBase> osbSet = new HashSet<OriginalSourceBase>();
 670 //            if(issTmp2!=null) {
 671 //                osbSet.addAll(issTmp2);
 672 //            }
 673 //            if(issTmp!=null) {
 674 //                osbSet.addAll(issTmp);
 675 //            }
 676
 677             IdentifiableSource sour = getIdentifiableSource(state.getRef(),null);
 678             String idInSource = derivedUnitFacade.getAccessionNumber() != null? derivedUnitFacade.getAccessionNumber():derivedUnitFacade.getCatalogNumber();
 679             sour.getCitation().setUri(state.getActualAccessPoint());
 680             sour.setIdInSource(idInSource);
 681             try{
 682                 if (sour.getCitation() != null){
 683                     if(StringUtils.isNotBlank(sour.getCitationMicroReference())) {
 684                         state.getDataHolder().getDocSources().add(sour.getCitation().getTitleCache()+ "---"+sour.getCitationMicroReference());
 685                     } else {
 686                         state.getDataHolder().getDocSources().add(sour.getCitation().getTitleCache());
 687                     }
 688                 }
 689             }catch(Exception e){
 690                 logger.warn("oups");
 691             }
 692
 693            derivedUnitFacade.addSource(sour);
 694           // sourceMap.put(sour.getCitation().getTitleCache()+ "---"+sour.getCitationMicroReference(),sour);
 695
 696 //            if( state.getConfig().isInteractWithUser()){
 697 //                List<OriginalSourceBase<?>>sources=null;
 698 //                if(!state.isDerivedUnitSourcesSet()){
 699 //                    sources= sui.askForSource(sourceMap, "the unit itself","",getReferenceService(), state.getDataHolder().getDocSources());
 700 //                    state.setDerivedUnitSources(sources);
 701 //                    state.setDerivedUnitSourcesSet(true);
 702 //                }
 703 //                else{
 704 //                    sources=state.getDerivedUnitSources();
 705 //                }
 706 ////                for (OriginalSourceBase<?> source:sources){
 707 ////                    if(source.isInstanceOf(IdentifiableSource.class)){
 708 ////                        if(sourceNotLinkedToElement(derivedUnitFacade,source)) {
 709 ////                            derivedUnitFacade.addSource((IdentifiableSource)source.clone());
 710 ////                        }
 711 ////                    }else{
 712 ////                        if(sourceNotLinkedToElement(derivedUnitFacade,sour)) {
 713 ////                            derivedUnitFacade.addSource(OriginalSourceType.Import,source.getCitation(),source.getCitationMicroReference(), ioName);
 714 ////                        }
 715 ////                    }
 716 ////                }
 717 //            }else{
 718 //                for (OriginalSourceBase<?> sr : sourceMap.values()){
 719 //                    if(sr.isInstanceOf(IdentifiableSource.class)){
 720 //                        if(sourceNotLinkedToElement(derivedUnitFacade,sr)) {
 721 //                            derivedUnitFacade.addSource((IdentifiableSource)sr.clone());
 722 //                        }
 723 //                    }else{
 724 //                        if(sourceNotLinkedToElement(derivedUnitFacade,sr)) {
 725 //                            derivedUnitFacade.addSource(OriginalSourceType.Import,sr.getCitation(),sr.getCitationMicroReference(), ioName);
 726 //                        }
 727 //                    }
 728 //                }
 729 //            }
 730
 731             save(state.getDerivedUnitBase(), state);
 732
 733             if(logger.isDebugEnabled()) {
 734                 logger.info("saved ABCD specimen ...");
 735             }
 736
 737             // handle identifications
 738             handleIdentifications(state, derivedUnitFacade);
 739
 740             //associatedUnits
 741             if (handleAssociatedUnits){
 742                 importAssociatedUnits(state, item, derivedUnitFacade);
 743             }
 744             //siblings/ other children
 745             if (derivedUnitFacade.getType() != null && (derivedUnitFacade.getType().equals(SpecimenOrObservationType.LivingSpecimen) ||  derivedUnitFacade.getType().equals(SpecimenOrObservationType.TissueSample)  ||  derivedUnitFacade.getType().equals(SpecimenOrObservationType.OtherSpecimen)) &&state.getConfig().isGetSiblings()){
 746                 getSiblings(state, item, derivedUnitFacade);
 747             }
 748
 749
 750         } catch (Exception e) {
 751             String message = "Error when reading record!";
 752             logger.warn(message);
 753             state.getReport().addException(message, e);
 754             e.printStackTrace();
 755             state.setUnsuccessfull();
 756         }
 757
 758         return;
 759     }
 760
 761     @Override
 762     protected void importAssociatedUnits(Abcd206ImportState state, Object itemObject, DerivedUnitFacade derivedUnitFacade) {
 763
 764         Abcd206ImportConfigurator config = state.getConfig();
 765         //import associated units
 766         FieldUnit currentFieldUnit = derivedUnitFacade.innerFieldUnit();
 767         //TODO: push state (think of implementing stack architecture for state
 768         DerivedUnit currentUnit = state.getDerivedUnitBase();
 769         DerivationEvent currentDerivedFrom = currentUnit.getDerivedFrom();
 770         URI currentAccessPoint = state.getActualAccessPoint();
 771         String currentPrefix = state.getPrefix();
 772         Element item = null;
 773         if (itemObject instanceof Element){
 774             item = (Element)itemObject;
 775         }
 776         NodeList unitAssociationList = null;
 777         if (item != null){
 778             unitAssociationList = item.getElementsByTagName(currentPrefix+"UnitAssociation");
 779             for(int k=0;k<unitAssociationList.getLength();k++){
 780                 if(unitAssociationList.item(k) instanceof Element){
 781                     Element unitAssociation = (Element)unitAssociationList.item(k);
 782                     UnitAssociationParser unitAssociationParser = new UnitAssociationParser(currentPrefix, state.getReport(), state.getCdmRepository());
 783                     UnitAssociationWrapper associationWrapper = unitAssociationParser.parse(unitAssociation);
 784
 785                     state.setActualAccessPoint(associationWrapper.getAccesPoint());
 786                     if(associationWrapper!=null){
 787                         NodeList associatedUnits = associationWrapper.getAssociatedUnits();
 788                         if(associatedUnits!=null){
 789                             for(int m=0;m<associatedUnits.getLength();m++){
 790                                 if(associatedUnits.item(m) instanceof Element){
 791                                     state.reset();
 792                                     state.setPrefix(associationWrapper.getPrefix());
 793                                     Abcd206ImportParser.setUnitPropertiesXML((Element) associatedUnits.item(m), new Abcd206XMLFieldGetter(state.getDataHolder(), state.getPrefix()), state);
 794                                     logger.debug("derived unit: " + state.getDerivedUnitBase().toString() + " associated unit: " +state.getDataHolder().getKindOfUnit() + ", " + state.getDataHolder().accessionNumber + ", " + state.getDataHolder().getRecordBasis() + ", " + state.getDataHolder().getUnitID());
 795                                     handleSingleUnit(state, associatedUnits.item(m), true);
 796
 797                                     DerivedUnit associatedUnit = state.getDerivedUnitBase();
 798                                     FieldUnit associatedFieldUnit = null;
 799                                     java.util.Collection<FieldUnit> associatedFieldUnits = state.getCdmRepository().getOccurrenceService().getFieldUnits(associatedUnit.getUuid());
 800                                     //ignore field unit if associated unit has more than one
 801                                     if(associatedFieldUnits.size()>1){
 802                                         state.getReport().addInfoMessage(String.format("%s has more than one field unit.", associatedUnit));
 803                                     }
 804                                     else if(associatedFieldUnits.size()==1){
 805                                         associatedFieldUnit = associatedFieldUnits.iterator().next();
 806                                     }
 807
 808                                     //attach current unit and associated unit depending on association type
 809
 810                                     //parent-child relation:
 811                                     //copy derivation event and connect parent and sub derivative
 812 //                                    if(associationWrapper.getAssociationType().contains("individual") || associationWrapper.getAssociationType().contains("culture") || associationWrapper.getAssociationType().contains("sample") || associationWrapper.getAssociationType().contains("same in situ")){
 813                                         if(currentDerivedFrom==null){
 814                                             state.getReport().addInfoMessage(String.format("No derivation event found for unit %s. Defaulting to ACCESSIONING event.",SpecimenImportUtility.getUnitID(currentUnit, config)));
 815                                             DerivationEvent.NewSimpleInstance(associatedUnit, currentUnit, DerivationEventType.ACCESSIONING());
 816                                         }
 817                                         else{
 818                                             DerivationEvent updatedDerivationEvent = DerivationEvent.NewSimpleInstance(associatedUnit, currentUnit, currentDerivedFrom.getType());
 819                                             updatedDerivationEvent.setActor(currentDerivedFrom.getActor());
 820                                             updatedDerivationEvent.setDescription(currentDerivedFrom.getDescription());
 821                                             updatedDerivationEvent.setInstitution(currentDerivedFrom.getInstitution());
 822                                             updatedDerivationEvent.setTimeperiod(currentDerivedFrom.getTimeperiod());
 823
 824                                         }
 825                                         state.getReport().addDerivate(associatedUnit, currentUnit, config);
 826 //                                    }
 827                                     //siblings relation
 828                                     //connect current unit to field unit of associated unit
 829 //                                    else if(associationWrapper.getAssociationType().contains("population")|| associationWrapper.getAssociationType().contains("sample")){
 830 //                                        //no associated field unit -> using current one
 831 //                                        if(associatedFieldUnit==null){
 832 //                                            if(currentFieldUnit!=null){
 833 //                                                DerivationEvent.NewSimpleInstance(currentFieldUnit, associatedUnit, DerivationEventType.ACCESSIONING());
 834 //                                            }
 835 //                                        }
 836 //                                        else{
 837 //                                            if(currentDerivedFrom==null){
 838 //                                                state.getReport().addInfoMessage("No derivation event found for unit "+SpecimenImportUtility.getUnitID(currentUnit, config)+". Defaulting to ACCESIONING event.");
 839 //                                                DerivationEvent.NewSimpleInstance(associatedFieldUnit, currentUnit, DerivationEventType.ACCESSIONING());
 840 //                                            }
 841 //                                            if(currentDerivedFrom!=null && associatedFieldUnit==currentFieldUnit){
 842 //                                                DerivationEvent updatedDerivationEvent = DerivationEvent.NewSimpleInstance(associatedFieldUnit, currentUnit, currentDerivedFrom.getType());
 843 //                                                updatedDerivationEvent.setActor(currentDerivedFrom.getActor());
 844 //                                                updatedDerivationEvent.setDescription(currentDerivedFrom.getDescription());
 845 //                                                updatedDerivationEvent.setInstitution(currentDerivedFrom.getInstitution());
 846 //                                                updatedDerivationEvent.setTimeperiod(currentDerivedFrom.getTimeperiod());
 847 //                                            }
 848 //                                        }
 849 //                                    }
 850
 851                                     //delete current field unit if replaced
 852                                     if(currentFieldUnit!=null && currentDerivedFrom!=null
 853                                             && currentFieldUnit.getDerivationEvents().size()==1  && currentFieldUnit.getDerivationEvents().contains(currentDerivedFrom) //making sure that the field unit
 854                                             && currentDerivedFrom.getDerivatives().size()==1 && currentDerivedFrom.getDerivatives().contains(currentUnit) //is not attached to other derived units
 855                                             && currentDerivedFrom!=currentUnit.getDerivedFrom() // <- derivation has been replaced and can be deleted
 856                                             ){
 857                                         currentFieldUnit.removeDerivationEvent(currentDerivedFrom);
 858                                         state.getCdmRepository().getOccurrenceService().delete(currentFieldUnit);
 859                                          }
 860
 861                                     save(associatedUnit, state);
 862                                 }
 863                             }
 864                         }
 865                     }
 866                 }
 867             }
 868         }
 869         //TODO: pop state
 870         state.reset();
 871         state.setDerivedUnitBase(currentUnit);
 872         state.setActualAccessPoint(currentAccessPoint);
 873         state.setPrefix(currentPrefix);
 874     }
 875
 876
 877
 878
 879     /**
 880      * @param derivedUnitFacade
 881      * @param sour
 882      * @return
 883      */
 884     private boolean sourceNotLinkedToElement(DerivedUnitFacade derivedUnitFacade, OriginalSourceBase<?> source) {
 885         Set<IdentifiableSource> linkedSources = derivedUnitFacade.getSources();
 886         for (IdentifiableSource is:linkedSources){
 887             if (is.getCitation()!=null && source.getCitation()!=null &&
 888                     is.getCitation().getTitleCache().equalsIgnoreCase(source.getCitation().getTitleCache())){
 889                 String isDetail =  is.getCitationMicroReference();
 890                 if ((StringUtils.isBlank(isDetail) && StringUtils.isBlank(source.getCitationMicroReference()))
 891                         || (isDetail != null && isDetail.equalsIgnoreCase(source.getCitationMicroReference())) ) {
 892                     return false;
 893                 }
 894             }
 895         }
 896         return true;
 897     }
 898
 899
 900
 901     //    /**
 902     //     * @param reference
 903     //     * @param citationDetail
 904     //     * @return
 905     //     */
 906     //    private DescriptionElementSource getDescriptionSource(Reference reference, String citationDetail) {
 907     //
 908     //        List<OriginalSourceBase> issTmp2 = getCommonService().list(DescriptionElementSource.class, null, null, null, null);
 909     //
 910     //        try {
 911     //            for (OriginalSourceBase<?> osb:issTmp2){
 912     //                if (osb.getCitation().equals(reference) && osb.getCitationMicroReference().equalsIgnoreCase(citationDetail)) {
 913     //                    return (DescriptionElementSource) osb.clone();
 914     //                }
 915     //            }
 916     //        } catch (CloneNotSupportedException e) {
 917     //            // TODO Auto-generated catch block
 918     //            e.printStackTrace();
 919     //        }
 920     //
 921     //        DescriptionElementSource sour = DescriptionElementSource.NewInstance(OriginalSourceType.Import,null,null, reference,citationDetail);
 922     //        return sour;
 923     //    }
 924
 925
 926
 927
 928     /**
 929      * setCollectionData : store the collection object into the
 930      * derivedUnitFacade
 931      *
 932      * @param state
 933      */
 934     protected void setCollectionData(Abcd206ImportState state, DerivedUnitFacade derivedUnitFacade) {
 935         Abcd206ImportConfigurator config = state.getConfig();
 936         SpecimenImportUtility.setUnitID(derivedUnitFacade.innerDerivedUnit(), state.getDataHolder().getUnitID(), config);
 937         if(!config.isMapUnitIdToAccessionNumber()){
 938             derivedUnitFacade.setAccessionNumber(NB(state.getDataHolder().accessionNumber));
 939         }
 940         // derivedUnitFacade.setCollectorsNumber(NB(state.getDataHolder().collectorsNumber));
 941
 942         /*
 943          * INSTITUTION & COLLECTION
 944          */
 945         // manage institution
 946         Institution institution = this.getInstitution(NB(state.getDataHolder().institutionCode), state);
 947         // manage collection
 948         Collection collection = this.getCollection(institution, NB(state.getDataHolder().collectionCode), state);
 949         // link specimen & collection
 950         derivedUnitFacade.setCollection(collection);
 951     }
 952
 953     /**
 954      * getFacade : get the DerivedUnitFacade based on the recordBasis
 955      * @param state
 956      *
 957      * @return DerivedUnitFacade
 958      */
 959     @Override
 960     protected DerivedUnitFacade getFacade(Abcd206ImportState state) {
 961         if(logger.isDebugEnabled()) {
 962             logger.info("getFacade()");
 963         }
 964         SpecimenOrObservationType type = null;
 965         DefinedTerm kindOfUnit = null;
 966
 967         // create specimen
 968         if (NB((state.getDataHolder()).getRecordBasis()) != null) {
 969             if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("living")>-1) {
 970                 type = SpecimenOrObservationType.LivingSpecimen;
 971             }
 972             else if (state.getDataHolder().getRecordBasis().toLowerCase().startsWith("s") || state.getDataHolder().getRecordBasis().toLowerCase().indexOf("specimen")>-1) {// specimen
 973                 type = SpecimenOrObservationType.PreservedSpecimen;
 974             }
 975             else if (state.getDataHolder().getRecordBasis().toLowerCase().startsWith("o") ||state.getDataHolder().getRecordBasis().toLowerCase().indexOf("observation")>-1 ) {
 976                 type = SpecimenOrObservationType.Observation;
 977             }
 978             else if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("fossil")>-1){
 979                 type = SpecimenOrObservationType.Fossil;
 980             }else if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("materialsample")>-1){
 981                 type = SpecimenOrObservationType.OtherSpecimen;
 982             }
 983             else if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("sample")>-1){
 984                 type = SpecimenOrObservationType.TissueSample;
 985             }
 986
 987             if (type == null) {
 988                 logger.info("The basis of record does not seem to be known: " + state.getDataHolder().getRecordBasis());
 989                 type = SpecimenOrObservationType.DerivedUnit;
 990             }
 991         } else {
 992             logger.info("The basis of record is null");
 993             type = SpecimenOrObservationType.DerivedUnit;
 994         }
 995
 996         if (NB((state.getDataHolder()).getKindOfUnit()) != null) {
 997             if (state.getDataHolder().getKindOfUnit().toLowerCase().indexOf("clone")>-1) {
 998                 kindOfUnit = getKindOfUnit(state, null, "clone culture", "clone culture", "cc", null);
 999             }
1000             else if (state.getDataHolder().getKindOfUnit().toLowerCase().startsWith("live"))  {
1001                 kindOfUnit = getKindOfUnit(state, null, "live sample", "live sample", "ls", null);
1002             }
1003
1004
1005             if (kindOfUnit == null) {
1006                 logger.info("The kind of unit does not seem to be known: " + state.getDataHolder().getKindOfUnit());
1007
1008             }
1009         } else {
1010             logger.info("The kind of unit is null");
1011
1012         }
1013         DerivedUnitFacade derivedUnitFacade = DerivedUnitFacade.NewInstance(type);
1014         derivedUnitFacade.setFieldUnit(state.getFieldUnit(state.getDataHolder().getFieldNumber()));
1015         derivedUnitFacade.setKindOfUnit(kindOfUnit);
1016         return derivedUnitFacade;
1017     }
1018
1019     private void getCollectorsFromXML(Element root, Abcd206XMLFieldGetter abcdFieldGetter, Abcd206ImportState state) {
1020         NodeList group;
1021
1022         group = root.getChildNodes();
1023         for (int i = 0; i < group.getLength(); i++) {
1024             if (group.item(i).getNodeName().equals(state.getPrefix() + "Identifications")) {
1025                 group = group.item(i).getChildNodes();
1026                 break;
1027             }
1028         }
1029         //state.getDataHolder().gatheringAgents = "";
1030
1031         abcdFieldGetter.getType(root);
1032         abcdFieldGetter.getGatheringPeople(root);
1033     }
1034
1035
1036
1037     //    private void compareABCDtoCDM(URI urlFileName, List<String> knownElts, Abcd206XMLFieldGetter abcdFieldGetter) {
1038     //        try {
1039     //            DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
1040     //            DocumentBuilder constructeur = factory.newDocumentBuilder();
1041     //            URL url = urlFileName.toURL();
1042     //            Object o = url.getContent();
1043     //            InputStream is = (InputStream) o;
1044     //            Document document = constructeur.parse(is);
1045     //            Element root = document.getDocumentElement();
1046     //            abcdFieldGetter.traverse(root);
1047     //        }
1048     //        catch (ParserConfigurationException e){
1049     //            e.printStackTrace();
1050     //        }
1051     //        catch (SAXException e) {
1052     //            e.printStackTrace();
1053     //        }
1054     //        catch (IOException e) {
1055     //            e.printStackTrace();
1056     //        }
1057     //        Set<String> elts = state.getDataHolder().allABCDelements.keySet();
1058     //        Iterator<String> it = elts.iterator();
1059     //        String elt;
1060     //        while (it.hasNext()) {
1061     //            elt = it.next();
1062     //            if (knownElts.indexOf(elt) == -1) {
1063     //                if(DEBUG) {
1064     //                    logger.info("Unmerged ABCD element: " + elt + " - "+ state.getDataHolder().allABCDelements.get(elt));
1065     //                }
1066     //            }
1067     //        }
1068     //    }
1069
1070     /**
1071      * Load the list of names from the ABCD file and save them
1072      * @param state : the current ABCD import state
1073      * @param unitsList : the unit list from the ABCD file
1074      * @param abcdFieldGetter : the ABCD parser
1075      */
1076     private void prepareCollectors(Abcd206ImportState state, NodeList unitsList, Abcd206XMLFieldGetter abcdFieldGetter) {
1077
1078         TeamOrPersonBase<?> teamOrPerson = null;
1079
1080         //ImportHelper.setOriginalSource(teamOrPerson, state.getConfig().getSourceReference(), collector, "Collector");
1081         for (int i = 0; i < unitsList.getLength(); i++) {
1082             this.getCollectorsFromXML((Element) unitsList.item(i), abcdFieldGetter, state);
1083             if (!StringUtils.isBlank(state.getDataHolder().gatheringAgents)){
1084                 teamOrPerson = parseAuthorString(state.getDataHolder().gatheringAgents);
1085                 if (!state.getPersonStore().containsId(state.getDataHolder().gatheringAgents)) {
1086                     state.getPersonStore().put(state.getDataHolder().gatheringAgents, teamOrPerson);
1087                     if (logger.isDebugEnabled()) { logger.debug("Stored author " + state.getDataHolder().gatheringAgents  ); }
1088                 } else {
1089                     logger.warn("Not imported author with duplicated aut_id " + state.getDataHolder().gatheringAgents  );
1090                 }
1091             }
1092
1093         }
1094
1095
1096
1097
1098 //        List<String> collectorsU = new ArrayList<String>(new HashSet<String>(collectors));
1099 //        List<String> teamsU = new ArrayList<String>(new HashSet<String>(teams));
1100 //
1101 //
1102 //        //existing teams in DB
1103 //        Map<String,Team> titleCacheTeam = new HashMap<String, Team>();
1104        // List<UuidAndTitleCache<Team>> hiberTeam = new ArrayList<UuidAndTitleCache<Team>>();//getAgentService().getTeamUuidAndTitleCache();
1105
1106 //        Set<UUID> uuids = new HashSet<UUID>();
1107 //        for (UuidAndTitleCache<Team> hibernateT:hiberTeam){
1108 //            uuids.add(hibernateT.getUuid());
1109 //        }
1110 //        if (!uuids.isEmpty()){
1111 //            List<AgentBase> existingTeams = getAgentService().find(uuids);
1112 //            for (AgentBase<?> existingP:existingTeams){
1113 //                titleCacheTeam.put(existingP.getTitleCache(),CdmBase.deproxy(existingP,Team.class));
1114 //            }
1115 //        }
1116
1117
1118 //        Map<String,UUID> teamMap = new HashMap<String, UUID>();
1119 //        for (UuidAndTitleCache<Team> uuidt:hiberTeam){
1120 //            teamMap.put(uuidt.getTitleCache(), uuidt.getUuid());
1121 //        }
1122
1123         //existing persons in DB
1124 //        List<UuidAndTitleCache<Person>> hiberPersons = new ArrayList<UuidAndTitleCache<Person>>();//getAgentService().getPersonUuidAndTitleCache();
1125 //        Map<String,Person> titleCachePerson = new HashMap<String, Person>();
1126 //        uuids = new HashSet<UUID>();
1127 //        for (UuidAndTitleCache<Person> hibernateP:hiberPersons){
1128 //            uuids.add(hibernateP.getUuid());
1129 //        }
1130 //
1131 //        if (!uuids.isEmpty()){
1132 //            List<AgentBase> existingPersons = getAgentService().find(uuids);
1133 //            for (AgentBase<?> existingP:existingPersons){
1134 //                titleCachePerson.put(existingP.getTitleCache(),CdmBase.deproxy(existingP,Person.class));
1135 //            }
1136 //        }
1137 //
1138 //        Map<String,UUID> personMap = new HashMap<String, UUID>();
1139 //        for (UuidAndTitleCache<Person> person:hiberPersons){
1140 //            personMap.put(person.getTitleCache(), person.getUuid());
1141 //        }
1142 //
1143 //        java.util.Collection<Person> personToadd = new ArrayList<Person>();
1144 //        java.util.Collection<Team> teamToAdd = new ArrayList<Team>();
1145 //
1146 //        for (String collector:collectorsU){
1147 //            Person p = Person.NewInstance();
1148 //            p.setTitleCache(collector,true);
1149 //            if (!personMap.containsKey(p.getTitleCache())){
1150 //                personToadd.add(p);
1151 //            }
1152 //        }
1153 //        for (String team:teamsU){
1154 //            Team p = Team.NewInstance();
1155 //            p.setTitleCache(team,true);
1156 //            if (!teamMap.containsKey(p.getTitleCache())){
1157 //                teamToAdd.add(p);
1158 //            }
1159 //        }
1160 //
1161 //        if(!personToadd.isEmpty()){
1162 //            for (Person agent: personToadd){
1163 //                save(agent, state);
1164 //                titleCachePerson.put(agent.getTitleCache(),CdmBase.deproxy(agent, Person.class) );
1165 //            }
1166 //        }
1167 //
1168 //        Person ptmp ;
1169 //        Map <String,Integer>teamdone = new HashMap<String, Integer>();
1170 //        for (List<String> collteam: collectorinteams){
1171 //            if (!teamdone.containsKey(StringUtils.join(collteam.toArray(),"-"))){
1172 //                Team team = new Team();
1173 //                boolean em =true;
1174 //                for (String collector:collteam){
1175 //                    ptmp = Person.NewInstance();
1176 //                    ptmp.setTitleCache(collector,true);
1177 //                    Person p2 = titleCachePerson.get(ptmp.getTitleCache());
1178 //                    team.addTeamMember(p2);
1179 //                    em=false;
1180 //                }
1181 //                if (!em) {
1182 //                    teamToAdd.add(team);
1183 //                }
1184 //                teamdone.put(StringUtils.join(collteam.toArray(),"-"),0);
1185 //            }
1186 //        }
1187 //
1188 //        if(!teamToAdd.isEmpty()){
1189 //            for (Team agent: teamToAdd){
1190 //                save(agent, state);
1191 //                titleCacheTeam.put(agent.getTitleCache(), CdmBase.deproxy( agent,Team.class) );
1192 //            }
1193 //        }
1194
1195 //        ((Abcd206ImportConfigurator) state.getConfig()).setTeams(titleCacheTeam);
1196 //        ((Abcd206ImportConfigurator) state.getConfig()).setPersons(titleCachePerson);
1197     }
1198
1199
1200     @Override
1201     protected boolean doCheck(Abcd206ImportState state) {
1202         logger.warn("Checking not yet implemented for " + this.getClass().getSimpleName());
1203         return true;
1204     }
1205
1206     @Override
1207     protected boolean isIgnore(Abcd206ImportState state) {
1208         return false;
1209     }
1210
1211 }