Project

General

Profile

Download (62.2 KB) Statistics
| Branch: | Tag: | Revision:
1
/**
2
* Copyright (C) 2007 EDIT
3
* European Distributed Institute of Taxonomy
4
* http://www.e-taxonomy.eu
5
*
6
* The contents of this file are subject to the Mozilla Public License Version 1.1
7
* See LICENSE.TXT at the top of this package for the full license terms.
8
*/
9

    
10
package eu.etaxonomy.cdm.io.specimen;
11

    
12

    
13
import java.util.ArrayList;
14
import java.util.HashMap;
15
import java.util.HashSet;
16
import java.util.List;
17
import java.util.Map;
18
import java.util.Set;
19
import java.util.UUID;
20

    
21
import org.apache.log4j.Logger;
22

    
23
import eu.etaxonomy.cdm.api.application.ICdmApplicationConfiguration;
24
import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
25
import eu.etaxonomy.cdm.api.service.config.FindOccurrencesConfigurator;
26
import eu.etaxonomy.cdm.api.service.pager.Pager;
27
import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
28
import eu.etaxonomy.cdm.io.common.CdmImportBase;
29
import eu.etaxonomy.cdm.io.common.IImportConfigurator;
30
import eu.etaxonomy.cdm.io.specimen.abcd206.in.Identification;
31
import eu.etaxonomy.cdm.io.specimen.abcd206.in.SpecimenImportReport;
32
import eu.etaxonomy.cdm.model.agent.AgentBase;
33
import eu.etaxonomy.cdm.model.agent.Institution;
34
import eu.etaxonomy.cdm.model.agent.Team;
35
import eu.etaxonomy.cdm.model.common.CdmBase;
36
import eu.etaxonomy.cdm.model.common.ISourceable;
37
import eu.etaxonomy.cdm.model.common.IdentifiableSource;
38
import eu.etaxonomy.cdm.model.common.LanguageString;
39
import eu.etaxonomy.cdm.model.common.OriginalSourceBase;
40
import eu.etaxonomy.cdm.model.common.OriginalSourceType;
41
import eu.etaxonomy.cdm.model.description.DescriptionBase;
42
import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
43
import eu.etaxonomy.cdm.model.description.Feature;
44
import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
45
import eu.etaxonomy.cdm.model.description.TaxonDescription;
46
import eu.etaxonomy.cdm.model.name.BacterialName;
47
import eu.etaxonomy.cdm.model.name.BotanicalName;
48
import eu.etaxonomy.cdm.model.name.CultivarPlantName;
49
import eu.etaxonomy.cdm.model.name.INonViralName;
50
import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
51
import eu.etaxonomy.cdm.model.name.NonViralName;
52
import eu.etaxonomy.cdm.model.name.Rank;
53
import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
54
import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
55
import eu.etaxonomy.cdm.model.name.TaxonNameBase;
56
import eu.etaxonomy.cdm.model.name.TaxonNameFactory;
57
import eu.etaxonomy.cdm.model.occurrence.Collection;
58
import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
59
import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
60
import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
61
import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationType;
62
import eu.etaxonomy.cdm.model.reference.Reference;
63
import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
64
import eu.etaxonomy.cdm.model.taxon.Classification;
65
import eu.etaxonomy.cdm.model.taxon.Synonym;
66
import eu.etaxonomy.cdm.model.taxon.Taxon;
67
import eu.etaxonomy.cdm.model.taxon.TaxonBase;
68
import eu.etaxonomy.cdm.model.taxon.TaxonNode;
69
import eu.etaxonomy.cdm.persistence.dto.UuidAndTitleCache;
70
import eu.etaxonomy.cdm.persistence.query.MatchMode;
71
import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
72
import eu.etaxonomy.cdm.strategy.parser.ParserProblem;
73

    
74

    
75
/**
76
 * @author p.kelbert
77
 * @created 20.10.2008
78
 */
79
public abstract class SpecimenImportBase<CONFIG extends IImportConfigurator, STATE extends SpecimenImportStateBase>  extends CdmImportBase<CONFIG, STATE> {
80

    
81
    private static final long serialVersionUID = 4423065367998125678L;
82
    private static final Logger logger = Logger.getLogger(SpecimenImportBase.class);
83
	protected final boolean DEBUG = true;
84

    
85
	protected static final UUID SPECIMEN_SCAN_TERM = UUID.fromString("acda15be-c0e2-4ea8-8783-b9b0c4ad7f03");
86

    
87
	private static final String COLON = ":";
88

    
89

    
90
	@Override
91
    protected abstract void doInvoke(STATE state);
92

    
93
	/**
94
	 * Handle a single unit
95
	 * @param state
96
	 * @param item
97
	 */
98
	protected abstract void handleSingleUnit(STATE state, Object item) ;
99

    
100

    
101

    
102
	protected TaxonNameBase<?, ?> getOrCreateTaxonName(String scientificName, Rank rank, boolean preferredFlag, STATE state, int unitIndexInAbcdFile){
103
        TaxonNameBase<?, ?> taxonName = null;
104
        SpecimenImportConfiguratorBase<?,?> config = state.getConfig();
105

    
106
        //check atomised name data for rank
107
        //new name will be created
108
        TaxonNameBase<?,?> atomisedTaxonName = null;
109
        if (rank==null && unitIndexInAbcdFile>=0 && ((state.getDataHolder().getAtomisedIdentificationList() != null && !state.getDataHolder().getAtomisedIdentificationList().isEmpty())|| state.getDataHolder().getAtomisedIdentificationList().size() > 0)) {
110
            atomisedTaxonName = setTaxonNameByType(state.getDataHolder().getAtomisedIdentificationList().get(unitIndexInAbcdFile), scientificName, state);
111
            if(atomisedTaxonName!=null){
112
                rank = atomisedTaxonName.getRank();
113
            }
114
        }
115
        if(config.isReuseExistingTaxaWhenPossible()){
116
            TaxonNameBase<?,?> parsedName = atomisedTaxonName;
117
            if(parsedName==null){
118
                parsedName = parseScientificName(scientificName, state, state.getReport());
119
            }
120
            atomisedTaxonName = parsedName;
121
            if(config.isIgnoreAuthorship() && parsedName!=null && preferredFlag){
122
                // do not ignore authorship for non-preferred names because they need
123
                // to be created for the determination history
124
                String nameCache = parsedName.getNameCache();
125
                List<NonViralName> names = getNameService().findNamesByNameCache(nameCache, MatchMode.EXACT, null);
126
                if (!names.isEmpty()){
127
                     return getBestMatchingName(scientificName, new ArrayList<TaxonNameBase>(names), state);
128
                }
129
            } else {
130
                //search for existing names
131
                List<TaxonNameBase> names = getNameService().listByTitle(TaxonNameBase.class, scientificName, MatchMode.EXACT, null, null, null, null, null);
132
                taxonName = getBestMatchingName(scientificName, names, state);
133
                //still nothing found -> try with the atomised name full title cache
134
                if(taxonName==null && atomisedTaxonName!=null){
135
                    names = getNameService().listByTitle(TaxonNameBase.class, atomisedTaxonName.getFullTitleCache(), MatchMode.EXACT, null, null, null, null, null);
136
                    taxonName = getBestMatchingName(atomisedTaxonName.getTitleCache(), names, state);
137
                    //still nothing found -> try with the atomised name title cache
138
                    if(taxonName==null){
139
                        names = getNameService().listByTitle(TaxonNameBase.class, atomisedTaxonName.getTitleCache(), MatchMode.EXACT, null, null, null, null, null);
140
                        taxonName = getBestMatchingName(atomisedTaxonName.getTitleCache(), names, state);
141
                    }
142
                }
143

    
144
            }
145

    
146
        }
147

    
148
        if(taxonName==null && atomisedTaxonName!=null){
149
            taxonName = atomisedTaxonName;
150
            state.getReport().addName(taxonName);
151
            logger.info("Created new taxon name "+taxonName);
152
            if(taxonName.hasProblem()){
153
                state.getReport().addInfoMessage(String.format("Created %s with parsing problems", taxonName));
154
            }
155
            if(!atomisedTaxonName.getTitleCache().equals(scientificName)){
156
                state.getReport().addInfoMessage(String.format("Taxon %s was parsed as %s", scientificName, atomisedTaxonName.getTitleCache()));
157
            }
158
        }
159
        else if(taxonName==null){
160
            //create new taxon name
161

    
162
            if (state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNAFP)){
163
                taxonName = TaxonNameFactory.NewBotanicalInstance(rank);
164
            }else if (state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICZN)){
165
                taxonName = TaxonNameFactory.NewZoologicalInstance(rank);
166
            }else{
167
                taxonName = TaxonNameFactory.NewNonViralInstance(rank);
168
            }
169
            taxonName.setFullTitleCache(scientificName,true);
170
            taxonName.setTitleCache(scientificName, true);
171
            state.getReport().addName(taxonName);
172
            logger.info("Created new taxon name "+taxonName);
173
        }
174
        save(taxonName, state);
175
        return taxonName;
176
    }
177

    
178
	 protected TaxonNameBase<?, ?> getBestMatchingName(String scientificName, java.util.Collection<TaxonNameBase> names, STATE state){
179
	        List<TaxonNameBase> namesWithAcceptedTaxa = new ArrayList<>();
180
	        for (TaxonNameBase name : names) {
181
	            if(!name.getTaxa().isEmpty()){
182
	                namesWithAcceptedTaxa.add(name);
183
	            }
184
	        }
185
	        String message = "More than one taxon name was found for "+scientificName+"!";
186
	        //check for names with accepted taxa
187
	        if(namesWithAcceptedTaxa.size()>0){
188
	            if(namesWithAcceptedTaxa.size()>1){
189
	                state.getReport().addInfoMessage(message);
190
	                logger.warn(message);
191
	                return null;
192
	            }
193
	            return namesWithAcceptedTaxa.iterator().next();
194
	        }
195
	        //no names with accepted taxa found -> check accepted taxa of synonyms
196
	        List<Taxon> taxaFromSynonyms = new ArrayList<>();
197
	        for (TaxonNameBase name : names) {
198
	            Set<TaxonBase> taxonBases = name.getTaxonBases();
199
	            for (TaxonBase taxonBase : taxonBases) {
200
	                if(taxonBase.isInstanceOf(Synonym.class)){
201
	                    Synonym synonym = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
202
	                    taxaFromSynonyms.add(synonym.getAcceptedTaxon());
203
	                }
204
	            }
205
	        }
206
	        if(taxaFromSynonyms.size()>0){
207
	            if(taxaFromSynonyms.size()>1){
208
	                state.getReport().addInfoMessage(message);
209
	                logger.warn(message);
210
	                return null;
211
	            }
212
	            return taxaFromSynonyms.iterator().next().getName();
213
	        }
214
	        return null;
215
	    }
216
	 /**
217
	     * Parse automatically the scientific name
218
	     * @param scientificName the scientific name to parse
219
	     * @param state the current import state
220
	     * @param report the import report
221
	     * @return a parsed name
222
	     */
223
	    protected TaxonNameBase<?,?> parseScientificName(String scientificName, STATE state, SpecimenImportReport report) {
224
	        NonViralNameParserImpl nvnpi = NonViralNameParserImpl.NewInstance();
225
	        TaxonNameBase<?,?> taxonName = null;
226
	        boolean problem = false;
227

    
228
	        if(DEBUG){
229
	            logger.info("parseScientificName " + state.getDataHolder().getNomenclatureCode().toString());
230
	        }
231

    
232
	        if (state.getDataHolder().getNomenclatureCode().toString().equals("Zoological") || state.getDataHolder().getNomenclatureCode().toString().contains("ICZN")) {
233
	            taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICZN, null);
234
	            if (taxonName.hasProblem()) {
235
	                problem = true;
236
	            }
237
	        }
238
	        else if (state.getDataHolder().getNomenclatureCode().toString().equals("Botanical") || state.getDataHolder().getNomenclatureCode().toString().contains("ICBN")) {
239
	            taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNAFP, null);
240
	            if (taxonName.hasProblem()) {
241
	                problem = true;
242
	            }
243
	        }
244
	        else if (state.getDataHolder().getNomenclatureCode().toString().equals("Bacterial") || state.getDataHolder().getNomenclatureCode().toString().contains("ICBN")) {
245
	            taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNB, null);
246
	            if (taxonName.hasProblem()) {
247
	                problem = true;
248
	            }
249
	        }
250
	        else if (state.getDataHolder().getNomenclatureCode().toString().equals("Cultivar") || state.getDataHolder().getNomenclatureCode().toString().contains("ICNCP")) {
251
	            taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNCP, null);
252
	            if (taxonName.hasProblem()) {
253
	                problem = true;
254
	            }
255
	        }
256
	        if (problem) {
257
	            String message = String.format("Parsing problems for %s", scientificName);
258
	            if(taxonName!=null){
259
	                for (ParserProblem parserProblem : taxonName.getParsingProblems()) {
260
	                    message += "\n\t- "+parserProblem;
261
	                }
262
	            }
263
	            report.addInfoMessage(message);
264
	            logger.info(message);
265
	        }
266
	        return taxonName;
267

    
268
	    }
269

    
270
	    /**
271
	     * Create the name without automatic parsing, either because it failed, or because the user deactivated it.
272
	     * The name is built upon the ABCD fields
273
	     * @param atomisedMap : the ABCD atomised fields
274
	     * @param fullName : the full scientific name
275
	     * @param state
276
	     * @return the corresponding Botanical or Zoological or... name
277
	     */
278
	    protected TaxonNameBase<?,?> setTaxonNameByType(
279
	            HashMap<String, String> atomisedMap, String fullName, STATE state) {
280
	        boolean problem = false;
281
	        if(DEBUG) {
282
	            logger.info("settaxonnamebytype " + state.getDataHolder().getNomenclatureCode().toString());
283
	        }
284

    
285
	        if (state.getDataHolder().getNomenclatureCode().equals("Zoological") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICZN.getUuid())) {
286
	            TaxonNameBase<?,?> taxonName = TaxonNameFactory.NewZoologicalInstance(null);
287
	            taxonName.setFullTitleCache(fullName, true);
288
	            taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
289
	            taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
290
	            taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap,"SpeciesEpithet")));
291
	            taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap,"SubspeciesEpithet")));
292

    
293
	            if (taxonName.getGenusOrUninomial() != null){
294
	                taxonName.setRank(Rank.GENUS());
295
	            }
296

    
297
	            if (taxonName.getInfraGenericEpithet() != null){
298
	                taxonName.setRank(Rank.SUBGENUS());
299
	            }
300

    
301
	            if (taxonName.getSpecificEpithet() != null){
302
	                taxonName.setRank(Rank.SPECIES());
303
	            }
304

    
305
	            if (taxonName.getInfraSpecificEpithet() != null){
306
	                taxonName.setRank(Rank.SUBSPECIES());
307
	            }
308

    
309
	            Team team = null;
310
	            if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
311
	                team = Team.NewInstance();
312
	                team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
313
	            }
314
	            else {
315
	                if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
316
	                    team = Team.NewInstance();
317
	                    team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
318
	                }
319
	            }
320
	            if (team != null) {
321
	                taxonName.setBasionymAuthorship(team);
322
	            }
323
	            else {
324
	                if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
325
	                    taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
326
	                }
327
	                else if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
328
	                    taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamAndYear"));
329
	                }
330
	            }
331
	            if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
332
	                team = Team.NewInstance();
333
	                team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
334
	                taxonName.setCombinationAuthorship(team);
335
	            }
336
	            if (taxonName.hasProblem()) {
337
	                logger.info("pb ICZN");
338
	                problem = true;
339
	            }
340
	            else {
341
	                return taxonName;
342
	            }
343
	        }
344
	        else if (state.getDataHolder().getNomenclatureCode().equals("Botanical") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNAFP.getUuid())) {
345
	            BotanicalName taxonName = (BotanicalName) parseScientificName(fullName, state, state.getReport());
346
	            if (taxonName != null){
347
	                return taxonName;
348
	            }
349
	            else{
350
	                taxonName = TaxonNameFactory.NewBotanicalInstance(null);
351
	            }
352
	            taxonName.setFullTitleCache(fullName, true);
353
	            taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
354
	            taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "FirstEpithet")));
355
	            taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "InfraSpeEpithet")));
356
	            try {
357
	                taxonName.setRank(Rank.getRankByName(getFromMap(atomisedMap, "Rank")));
358
	            } catch (Exception e) {
359
	                if (taxonName.getInfraSpecificEpithet() != null){
360
	                    taxonName.setRank(Rank.SUBSPECIES());
361
	                }
362
	                else if (taxonName.getSpecificEpithet() != null){
363
	                    taxonName.setRank(Rank.SPECIES());
364
	                }
365
	                else if (taxonName.getInfraGenericEpithet() != null){
366
	                    taxonName.setRank(Rank.SUBGENUS());
367
	                }
368
	                else if (taxonName.getGenusOrUninomial() != null){
369
	                    taxonName.setRank(Rank.GENUS());
370
	                }
371
	            }
372
	            Team team = null;
373
	            if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
374
	                team = Team.NewInstance();
375
	                team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
376
	                taxonName.setBasionymAuthorship(team);
377
	            }
378
	            if (getFromMap(atomisedMap, "AuthorTeam") != null) {
379
	                team = Team.NewInstance();
380
	                team.setTitleCache(getFromMap(atomisedMap, "AuthorTeam"), true);
381
	                taxonName.setCombinationAuthorship(team);
382
	            }
383
	            if (team == null) {
384
	                if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
385
	                    taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
386
	                }
387
	                else if (getFromMap(atomisedMap, "AuthorTeam") != null) {
388
	                    taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeam"));
389
	                }
390
	            }
391
	            if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
392
	                team = Team.NewInstance();
393
	                team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
394
	                taxonName.setCombinationAuthorship(team);
395
	            }
396
	            if (taxonName.hasProblem()) {
397
	                logger.info("pb ICBN");
398
	                problem = true;
399
	            }
400
	            else {
401
	                return taxonName;
402
	            }
403
	        }
404
	        else if (state.getDataHolder().getNomenclatureCode().equals("Bacterial") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNB.getUuid())) {
405
	            NonViralName<BacterialName> taxonName = TaxonNameFactory.NewBacterialInstance(null);
406
	            taxonName.setFullTitleCache(fullName, true);
407
	            taxonName.setGenusOrUninomial(getFromMap(atomisedMap, "Genus"));
408
	            taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
409
	            taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "Species")));
410
	            taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "SubspeciesEpithet")));
411

    
412
	            if (taxonName.getGenusOrUninomial() != null){
413
	                taxonName.setRank(Rank.GENUS());
414
	            }
415
	            else if (taxonName.getInfraGenericEpithet() != null){
416
	                taxonName.setRank(Rank.SUBGENUS());
417
	            }
418
	            else if (taxonName.getSpecificEpithet() != null){
419
	                taxonName.setRank(Rank.SPECIES());
420
	            }
421
	            else if (taxonName.getInfraSpecificEpithet() != null){
422
	                taxonName.setRank(Rank.SUBSPECIES());
423
	            }
424

    
425
	            if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
426
	                Team team = Team.NewInstance();
427
	                team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
428
	                taxonName.setCombinationAuthorship(team);
429
	            }
430
	            if (getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear") != null) {
431
	                Team team = Team.NewInstance();
432
	                team.setTitleCache(getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear"), true);
433
	                taxonName.setBasionymAuthorship(team);
434
	            }
435
	            if (taxonName.hasProblem()) {
436
	                logger.info("pb ICNB");
437
	                problem = true;
438
	            }
439
	            else {
440
	                return taxonName;
441
	            }
442
	        }
443
	        else if (state.getDataHolder().getNomenclatureCode().equals("Cultivar")) {
444
	            CultivarPlantName taxonName = TaxonNameFactory.NewCultivarInstance(null);
445

    
446
	            if (taxonName.hasProblem()) {
447
	                logger.info("pb ICNCP");
448
	                problem = true;
449
	            }
450
	            else {
451
	                return taxonName;
452
	            }
453
	            return taxonName;
454
	        }
455

    
456
	        if (problem) {
457
	            logger.info("Problem im setTaxonNameByType ");
458
	            TaxonNameBase<?,?> taxonName = TaxonNameFactory.NewNonViralInstance(null);
459
	            taxonName.setFullTitleCache(fullName, true);
460
	            return taxonName;
461
	        }
462
	        TaxonNameBase<?,?> tn = TaxonNameFactory.NewNonViralInstance(null);
463
	        return tn;
464
	    }
465

    
466
	    /**
467
	     * Get a formated string from a hashmap
468
	     * @param atomisedMap
469
	     * @param key
470
	     * @return
471
	     */
472
	    private String getFromMap(HashMap<String, String> atomisedMap, String key) {
473
	        String value = null;
474
	        if (atomisedMap.containsKey(key)) {
475
	            value = atomisedMap.get(key);
476
	        }
477

    
478
	        try {
479
	            if (value != null && key.matches(".*Year.*")) {
480
	                value = value.trim();
481
	                if (value.matches("[a-z A-Z ]*[0-9]{4}$")) {
482
	                    String tmp = value.split("[0-9]{4}$")[0];
483
	                    int year = Integer.parseInt(value.split(tmp)[1]);
484
	                    if (year >= 1752) {
485
	                        value = tmp;
486
	                    }
487
	                    else {
488
	                        value = null;
489
	                    }
490
	                }
491
	                else {
492
	                    value = null;
493
	                }
494
	            }
495
	        }
496
	        catch (Exception e) {
497
	            value = null;
498
	        }
499
	        return value;
500
	    }
501

    
502
	    /**
503
	     * Very fast and dirty implementation to allow handling of transient objects as described in
504
	     * https://dev.e-taxonomy.eu/trac/ticket/3726
505
	     *
506
	     * Not yet complete.
507
	     *
508
	     * @param cdmBase
509
	     * @param state
510
	     */
511
	    protected UUID save(CdmBase cdmBase, SpecimenImportStateBase state) {
512
	        ICdmApplicationConfiguration cdmRepository = state.getConfig().getCdmAppController();
513
	        if (cdmRepository == null){
514
	            cdmRepository = this;
515
	        }
516

    
517
	        if (cdmBase.isInstanceOf(LanguageString.class)){
518
	            return cdmRepository.getTermService().saveLanguageData(CdmBase.deproxy(cdmBase, LanguageString.class));
519
	        }else if (cdmBase.isInstanceOf(SpecimenOrObservationBase.class)){
520
	            return cdmRepository.getOccurrenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, SpecimenOrObservationBase.class));
521
	        }else if (cdmBase.isInstanceOf(Reference.class)){
522
	            return cdmRepository.getReferenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, Reference.class));
523
	        }else if (cdmBase.isInstanceOf(Classification.class)){
524
	            return cdmRepository.getClassificationService().saveOrUpdate(CdmBase.deproxy(cdmBase, Classification.class));
525
	        }else if (cdmBase.isInstanceOf(AgentBase.class)){
526
	            return cdmRepository.getAgentService().saveOrUpdate(CdmBase.deproxy(cdmBase, AgentBase.class));
527
	        }else if (cdmBase.isInstanceOf(Collection.class)){
528
	            return cdmRepository.getCollectionService().saveOrUpdate(CdmBase.deproxy(cdmBase, Collection.class));
529
	        }else if (cdmBase.isInstanceOf(DescriptionBase.class)){
530
	            return cdmRepository.getDescriptionService().saveOrUpdate(CdmBase.deproxy(cdmBase, DescriptionBase.class));
531
	        }else if (cdmBase.isInstanceOf(TaxonBase.class)){
532
	            return cdmRepository.getTaxonService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonBase.class));
533
	        }else if (cdmBase.isInstanceOf(TaxonNameBase.class)){
534
	            return cdmRepository.getNameService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonNameBase.class));
535
	        }else if (cdmBase.isInstanceOf(TaxonNode.class)){
536
                return cdmRepository.getTaxonNodeService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonNode.class));
537
            }else{
538
	            throw new IllegalArgumentException("Class not supported in save method: " + CdmBase.deproxy(cdmBase, CdmBase.class).getClass().getSimpleName());
539
	        }
540

    
541
	    }
542

    
543

    
544
	    protected SpecimenOrObservationBase findExistingSpecimen(String unitId, SpecimenImportStateBase state){
545
	        ICdmApplicationConfiguration cdmAppController = state.getConfig().getCdmAppController();
546
	        if(cdmAppController==null){
547
	            cdmAppController = this;
548
	        }
549
	        FindOccurrencesConfigurator config = new FindOccurrencesConfigurator();
550
	        config.setSignificantIdentifier(unitId);
551
	        Pager<SpecimenOrObservationBase> existingSpecimens = cdmAppController.getOccurrenceService().findByTitle(config);
552
	        if(!existingSpecimens.getRecords().isEmpty()){
553
	            if(existingSpecimens.getRecords().size()==1){
554
	                return existingSpecimens.getRecords().iterator().next();
555
	            }
556
	        }
557
	        return null;
558
	    }
559

    
560
	    protected abstract void importAssociatedUnits(STATE state, Object item, DerivedUnitFacade derivedUnitFacade);
561

    
562
	    /**
563
	     * getFacade : get the DerivedUnitFacade based on the recordBasis
564
	     * @param state
565
	     *
566
	     * @return DerivedUnitFacade
567
	     */
568
	    protected DerivedUnitFacade getFacade(STATE state) {
569
	        if(DEBUG) {
570
	            logger.info("getFacade()");
571
	        }
572
	        SpecimenOrObservationType type = null;
573

    
574
	        // create specimen
575
	        if (NB((state.getDataHolder().getRecordBasis())) != null) {
576
	            if (state.getDataHolder().getRecordBasis().toLowerCase().startsWith("s") || state.getDataHolder().getRecordBasis().toLowerCase().indexOf("specimen")>-1) {// specimen
577
	                type = SpecimenOrObservationType.PreservedSpecimen;
578
	            }
579
	            if (state.getDataHolder().getRecordBasis().toLowerCase().startsWith("o") ||state.getDataHolder().getRecordBasis().toLowerCase().indexOf("observation")>-1 ) {
580
	                type = SpecimenOrObservationType.Observation;
581
	            }
582
	            if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("fossil")>-1){
583
	                type = SpecimenOrObservationType.Fossil;
584
	            }
585
	            if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("living")>-1) {
586
	                type = SpecimenOrObservationType.LivingSpecimen;
587
	            }
588
	            if (type == null) {
589
	                logger.info("The basis of record does not seem to be known: " + state.getDataHolder().getRecordBasis());
590
	                type = SpecimenOrObservationType.DerivedUnit;
591
	            }
592
	            // TODO fossils?
593
	        } else {
594
	            logger.info("The basis of record is null");
595
	            type = SpecimenOrObservationType.DerivedUnit;
596
	        }
597
	        DerivedUnitFacade derivedUnitFacade = DerivedUnitFacade.NewInstance(type);
598
	        return derivedUnitFacade;
599
	    }
600

    
601
	    /**
602
	     * Look if the Institution does already exist
603
	     * @param institutionCode: a string with the institutioncode
604
	     * @param config : the configurator
605
	     * @return the Institution (existing or new)
606
	     */
607
	    protected Institution getInstitution(String institutionCode, STATE state) {
608
	        SpecimenImportConfiguratorBase config = state.getConfig();
609
	        Institution institution=null;
610
	        institution = (Institution)state.institutions.get(institutionCode);
611
	        if (institution != null){
612
	            return institution;
613
	        }
614
	        List<Institution> institutions;
615
	        try {
616
	            institutions = getAgentService().searchInstitutionByCode(institutionCode);
617

    
618
	        } catch (Exception e) {
619
	            institutions = new ArrayList<Institution>();
620
	            logger.warn(e);
621
	        }
622
	        if (institutions.size() > 0 && config.isReuseExistingMetaData()) {
623
	            for (Institution institut:institutions){
624
	                try{
625
	                    if (institut.getCode().equalsIgnoreCase(institutionCode)) {
626
	                        institution=institut;
627
	                        break;
628
	                    }
629
	                }catch(Exception e){logger.warn("no institution code in the db");}
630
	            }
631
	        }
632
	        if(DEBUG) {
633
	            if(institution !=null) {
634
	                logger.info("getinstitution " + institution.toString());
635
	            }
636
	        }
637
	        if (institution == null){
638
	            // create institution
639
	            institution = Institution.NewInstance();
640
	            institution.setCode(institutionCode);
641
	            institution.setTitleCache(institutionCode, true);
642
	            UUID uuid = save(institution, state);
643
	        }
644

    
645

    
646
	        state.institutions.put(institutionCode, institution);
647
	        return institution;
648
	    }
649

    
650
	    /**
651
	     * Look if the Collection does already exist
652
	     * @param collectionCode
653
	     * @param collectionCode: a string
654
	     * @param config : the configurator
655
	     * @return the Collection (existing or new)
656
	     */
657
	    protected Collection getCollection(Institution institution, String collectionCode, STATE state) {
658
	        SpecimenImportConfiguratorBase config = state.getConfig();
659
	        Collection collection = null;
660
	        List<Collection> collections;
661
	        collection = (Collection) state.collections.get(collectionCode);
662
	        if (collection != null){
663
	            return collection;
664
	        }
665
	        try {
666
	            collections = getCollectionService().searchByCode(collectionCode);
667
	        } catch (Exception e) {
668
	            collections = new ArrayList<Collection>();
669
	        }
670
	        if (collections.size() > 0 && config.isReuseExistingMetaData()) {
671
	            for (Collection coll:collections){
672
	                if (coll.getCode() != null && coll.getInstitute() != null
673
	                        && coll.getCode().equalsIgnoreCase(collectionCode) && coll.getInstitute().equals(institution)) {
674
	                    collection = coll;
675
	                    break;
676
	                }
677
	            }
678
	        }
679

    
680
	        if(collection == null){
681
	            collection =Collection.NewInstance();
682
	            collection.setCode(collectionCode);
683
	            collection.setInstitute(institution);
684
	            collection.setTitleCache(collectionCode);
685
	            UUID uuid = save(collection, state);
686
	        }
687

    
688

    
689

    
690
	        state.collections.put(collectionCode, collection);
691

    
692
	        return collection;
693
	    }
694

    
695
	    /**
696
	     * @param reference
697
	     * @param citationDetail
698
	     * @return
699
	     */
700
	    //FIXME this method is highly critical, because
701
	    //  * it will have serious performance and memory problems with large databases
702
	    //        (databases may easily have >1 Mio source records)
703
	    //  * it does not make sense to search for existing sources and then clone them
704
	    //    we need to search for existing references instead and use them (if exist)
705
	    //    for our new source.
706
	    protected IdentifiableSource getIdentifiableSource(Reference reference, String citationDetail) {
707

    
708
	      /*  List<IdentifiableSource> issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
709

    
710

    
711
	        if (reference != null){
712
	            try {
713
	                for (OriginalSourceBase<?> osb: issTmp){
714
	                    if (osb.getCitation() != null && osb.getCitation().getTitleCache().equalsIgnoreCase(reference.getTitleCache())){
715
	                        String osbDetail = osb.getCitationMicroReference();
716
	                        if ((StringUtils.isBlank(osbDetail) && StringUtils.isBlank(citationDetail))
717
	                                || (osbDetail != null && osbDetail.equalsIgnoreCase(citationDetail)) ) {
718
//	                            System.out.println("REFERENCE FOUND RETURN EXISTING SOURCE");
719
	                            return (IdentifiableSource) osb.clone();
720
	                        }
721
	                    }
722
	                }
723
	            } catch (CloneNotSupportedException e) {
724
	                throw new RuntimeException(e);
725
	            } catch (Exception e1){
726
	                e1.printStackTrace();
727
	            }
728
	        }
729
	*/
730
	        IdentifiableSource sour = IdentifiableSource.NewInstance(OriginalSourceType.Import,null,null, reference,citationDetail);
731
	        return sour;
732
	    }
733

    
734
	    /**
735
	     * Add the hierarchy for a Taxon(add higher taxa)
736
	     * @param classification
737
	     * @param taxon: a taxon to add as a node
738
	     * @param state: the ABCD import state
739
	     */
740
	    protected void addParentTaxon(Taxon taxon, STATE state, boolean preferredFlag, Classification classification){
741
	        INonViralName  nvname = taxon.getName();
742
	        Rank rank = nvname.getRank();
743
	        Taxon genus =null;
744
	        Taxon subgenus =null;
745
	        Taxon species = null;
746
	        Taxon subspecies = null;
747
	        Taxon parent = null;
748
	        if(rank!=null){
749
	            if (rank.isLower(Rank.GENUS() )){
750
	                String genusOrUninomial = nvname.getGenusOrUninomial();
751
	                TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(genusOrUninomial, Rank.GENUS(), preferredFlag, state, -1);
752
	                genus = getOrCreateTaxonForName(taxonName, state);
753
	                if (preferredFlag) {
754
	                    parent = linkParentChildNode(null, genus, classification, state);
755
	                }
756

    
757
	            }
758
	            if (rank.isLower(Rank.SUBGENUS())){
759
	                String prefix = nvname.getGenusOrUninomial();
760
	                String name = nvname.getInfraGenericEpithet();
761
	                if (name != null){
762
	                    TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(prefix+" "+name, Rank.SUBGENUS(), preferredFlag, state, -1);
763
	                    subgenus = getOrCreateTaxonForName(taxonName, state);
764
	                    if (preferredFlag) {
765
	                        parent = linkParentChildNode(genus, subgenus, classification, state);
766
	                    }            }
767
	            }
768
	            if (rank.isLower(Rank.SPECIES())){
769
	                if (subgenus!=null){
770
	                    String prefix = nvname.getGenusOrUninomial();
771
	                    String name = nvname.getInfraGenericEpithet();
772
	                    String spe = nvname.getSpecificEpithet();
773
	                    if (spe != null){
774
	                        TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(prefix+" "+name+" "+spe, Rank.SPECIES(), preferredFlag, state, -1);
775
	                        species = getOrCreateTaxonForName(taxonName, state);
776
	                        if (preferredFlag) {
777
	                            parent = linkParentChildNode(subgenus, species, classification, state);
778
	                        }
779
	                    }
780
	                }
781
	                else{
782
	                    String prefix = nvname.getGenusOrUninomial();
783
	                    String name = nvname.getSpecificEpithet();
784
	                    if (name != null){
785
	                        TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(prefix+" "+name, Rank.SPECIES(), preferredFlag, state, -1);
786
	                        species = getOrCreateTaxonForName(taxonName, state);
787
	                        if (preferredFlag) {
788
	                            parent = linkParentChildNode(genus, species, classification, state);
789
	                        }
790
	                    }
791
	                }
792
	            }
793
	            if (rank.isLower(Rank.INFRASPECIES())){
794
	                TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(nvname.getFullTitleCache(), Rank.SUBSPECIES(), preferredFlag, state, -1);
795
	                subspecies = getOrCreateTaxonForName(taxonName, state);
796
	                if (preferredFlag) {
797
	                    parent = linkParentChildNode(species, subspecies, classification, state);
798
	                }
799
	            }
800
	        }
801
	        if (preferredFlag && parent!=taxon) {
802
	            linkParentChildNode(parent, taxon, classification, state);
803
	        }
804
	    }
805

    
806
	    /**
807
	     * Link a parent to a child and save it in the current classification
808
	     * @param parent: the higher Taxon
809
	     * @param child : the lower (or current) Taxon
810
	     * return the Taxon from the new created Node
811
	     * @param classification
812
	     * @param state
813
	     */
814
	    protected Taxon linkParentChildNode(Taxon parent, Taxon child, Classification classification, STATE state) {
815
	        TaxonNode node =null;
816
	        if (parent != null) {
817
	            parent = (Taxon) getTaxonService().find(parent.getUuid());
818
	            child = (Taxon) getTaxonService().find(child.getUuid());
819
	            //here we do not have to check if the taxon nodes already exists
820
	            //this is done by classification.addParentChild()
821
	            //do not add child node if it already exists
822
	            if(hasTaxonNodeInClassification(child, classification)){
823
	                return child;
824
	            }
825
	            else{
826
	                node = classification.addParentChild(parent, child, state.getRef(), "");
827
	                save(node, state);
828
	            }
829
	        }
830
	        else {
831
	            child = (Taxon) getTaxonService().find(child.getUuid());
832
	            //do not add child node if it already exists
833
	            if(hasTaxonNodeInClassification(child, classification)){
834
	                return child;
835
	            }
836
	            else{
837
	                node = classification.addChildTaxon(child, state.getRef(), null);
838
	                save(node, state);
839
	            }
840
	        }
841
	        if(node!=null){
842
	            state.getReport().addTaxonNode(node);
843
	            return node.getTaxon();
844
	        }
845
	        String message = "Could not create taxon node for " +child;
846
	        state.getReport().addInfoMessage(message);
847
	        logger.warn(message);
848
	        return null;
849
	    }
850

    
851
	    protected Taxon getOrCreateTaxonForName(TaxonNameBase<?, ?> taxonNameBase, STATE state){
852

    
853
	        Set<Taxon> acceptedTaxa = taxonNameBase.getTaxa();
854
	        if(acceptedTaxa.size()>0){
855
	            Taxon firstAcceptedTaxon = acceptedTaxa.iterator().next();
856
	            if(acceptedTaxa.size()>1){
857
	                String message = "More than one accepted taxon was found for taxon name: "
858
	                        + taxonNameBase.getTitleCache() + "!\n" + firstAcceptedTaxon + "was chosen for "+state.getDerivedUnitBase();
859
	                state.getReport().addInfoMessage(message);
860
	                logger.warn(message);
861
	            }
862
	            else{
863
	                return firstAcceptedTaxon;
864
	            }
865
	        }
866
	        else{
867
	            Set<TaxonBase> taxonAndSynonyms = taxonNameBase.getTaxonBases();
868
	            for (TaxonBase taxonBase : taxonAndSynonyms) {
869
	                if(taxonBase.isInstanceOf(Synonym.class)){
870
	                    Synonym synonym = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
871
	                    Taxon acceptedTaxonOfSynonym = synonym.getAcceptedTaxon();
872
	                    if(acceptedTaxonOfSynonym == null){
873
	                        String message = "No accepted taxon could be found for taxon name: "
874
	                                + taxonNameBase.getTitleCache()
875
	                                + "!";
876
	                        state.getReport().addInfoMessage(message);
877
	                        logger.warn(message);
878
	                    }
879
	                    else{
880
	                        return acceptedTaxonOfSynonym;
881
	                    }
882
	                }
883
	            }
884
	        }
885
	        Taxon taxon = Taxon.NewInstance(taxonNameBase, state.getRef());
886
	        save(taxon, state);
887
	        state.getReport().addTaxon(taxon);
888
	        logger.info("Created new taxon "+ taxon);
889
	        return taxon;
890
	    }
891

    
892
	    private boolean hasTaxonNodeInClassification(Taxon taxon, Classification classification){
893
	        if(taxon.getTaxonNodes()!=null){
894
	            for (TaxonNode node : taxon.getTaxonNodes()){
895
	                if(node.getClassification().equals(classification)){
896
	                    return true;
897
	                }
898
	            }
899
	        }
900
	        return false;
901
	    }
902

    
903
	    /**
904
	     * HandleIdentifications : get the scientific names present in the ABCD
905
	     * document and store link them with the observation/specimen data
906
	     * @param state: the current ABCD import state
907
	     * @param derivedUnitFacade : the current derivedunitfacade
908
	     */
909
	    protected void handleIdentifications(STATE state, DerivedUnitFacade derivedUnitFacade) {
910
	       SpecimenImportConfiguratorBase config = state.getConfig();
911

    
912

    
913
	        String scientificName = "";
914
	        boolean preferredFlag = false;
915

    
916
	        if (state.getDataHolder().getNomenclatureCode() == ""){
917
	            if (config.getNomenclaturalCode() != null){
918
	                if (config.getNomenclaturalCode() != null){
919
	                    state.getDataHolder().setNomenclatureCode(config.getNomenclaturalCode().toString());
920
	                }
921
	            }
922
	        }
923

    
924
	        for (int i = 0; i < state.getDataHolder().getIdentificationList().size(); i++) {
925
	            Identification identification = state.getDataHolder().getIdentificationList().get(i);
926
	            scientificName = identification.getScientificName().replaceAll(" et ", " & ");
927

    
928
	            String preferred = identification.getPreferred();
929
	            if (preferred.equals("1") || preferred.toLowerCase().indexOf("true") != -1 || state.getDataHolder().getIdentificationList().size()==1) {
930
	                preferredFlag = true;
931
	            }
932
	            else {
933
	                preferredFlag = false;
934
	            }
935
	            if (identification.getCode() != null){
936
    	            if (identification.getCode().indexOf(':') != -1) {
937
    	                state.getDataHolder().setNomenclatureCode(identification.getCode().split(COLON)[1]);
938
    	            }
939
    	            else{
940
    	                state.getDataHolder().setNomenclatureCode(identification.getCode());
941
    	            }
942
	            }
943
	            TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(scientificName, null, preferredFlag, state, i);
944
	            Taxon taxon = getOrCreateTaxonForName(taxonName, state);
945
	            addTaxonNode(taxon, state,preferredFlag);
946
	            linkDeterminationEvent(state, taxon, preferredFlag, derivedUnitFacade);
947
	        }
948
	    }
949

    
950
	    /**
951
	     * @param taxon : a taxon to add as a node
952
	     * @param state : the ABCD import state
953
	     */
954
	    protected void addTaxonNode(Taxon taxon, STATE state, boolean preferredFlag) {
955
	        SpecimenImportConfiguratorBase config = state.getConfig();
956
	        logger.info("link taxon to a taxonNode "+taxon.getTitleCache());
957
	        //only add nodes if not already existing in current classification or default classification
958

    
959
	        //check if node exists in current classification
960
	        //NOTE: we cannot use hasTaxonNodeInClassification() here because we are first creating it here
961
	        if (!existsInClassification(taxon,state.getClassification(), state)){
962
	            if(config.isMoveNewTaxaToDefaultClassification()){
963
	                //check if node exists in default classification
964
	                if (!existsInClassification(taxon, state.getDefaultClassification(), state)){
965
	                    addParentTaxon(taxon, state, preferredFlag, state.getDefaultClassification());
966
	                }
967
	            }
968
	            else {
969
	                //add non-existing taxon to current classification
970
	                addParentTaxon(taxon, state, preferredFlag, state.getClassification());
971
	            }
972
	        }
973
	    }
974

    
975

    
976
	    private boolean existsInClassification(Taxon taxon, Classification classification, STATE state){
977
	        boolean exist = false;
978
	        ICdmApplicationConfiguration cdmAppController = state.getConfig().getCdmAppController();
979
            if(cdmAppController==null){
980
                cdmAppController = this;
981
            }
982
            if (classification != null){
983
    	        List<UuidAndTitleCache<TaxonNode>> uuidAndTitleCacheOfAllTaxa = cdmAppController.getClassificationService().getTaxonNodeUuidAndTitleCacheOfAcceptedTaxaByClassification(classification.getUuid());
984
    	        if (uuidAndTitleCacheOfAllTaxa != null){
985
        	        for (UuidAndTitleCache p : uuidAndTitleCacheOfAllTaxa){
986
        	            try{
987
        	                if(p.getTitleCache().equals(taxon.getTitleCache())) {
988
        	                    exist = true;
989
        	                }
990
        	            }
991
        	            catch(Exception e){
992
        	                logger.warn("TaxonNode doesn't seem to have a taxon");
993
        	            }
994
        	        }
995
    	        }
996
            }
997
	        return exist;
998
	    }
999

    
1000
	    /**
1001
	     * join DeterminationEvent to the Taxon Object
1002
	     * @param state : the ABCD import state
1003
	     * @param taxon: the current Taxon
1004
	     * @param preferredFlag :if the current name is preferred
1005
	     * @param derivedFacade : the derived Unit Facade
1006
	     */
1007
	    @SuppressWarnings("rawtypes")
1008
        protected void linkDeterminationEvent(STATE state, Taxon taxon, boolean preferredFlag,  DerivedUnitFacade derivedFacade) {
1009
	        SpecimenImportConfiguratorBase config = state.getConfig();
1010
	        if(DEBUG){
1011
	            logger.info("start linkdetermination with taxon:" + taxon.getUuid()+", "+taxon);
1012
	        }
1013

    
1014
	        DeterminationEvent determinationEvent = DeterminationEvent.NewInstance();
1015
	        determinationEvent.setTaxonName(taxon.getName());
1016
	        determinationEvent.setPreferredFlag(preferredFlag);
1017

    
1018
	        determinationEvent.setIdentifiedUnit(state.getDerivedUnitBase());
1019
	        state.getDerivedUnitBase().addDetermination(determinationEvent);
1020

    
1021
	        if(DEBUG){
1022
	            logger.info("NB TYPES INFO: "+ state.getDataHolder().getStatusList().size());
1023
	        }
1024
	        for (SpecimenTypeDesignationStatus specimenTypeDesignationstatus : state.getDataHolder().getStatusList()) {
1025
	            if (specimenTypeDesignationstatus != null) {
1026
	                if(DEBUG){
1027
	                    logger.info("specimenTypeDesignationstatus :"+ specimenTypeDesignationstatus);
1028
	                }
1029

    
1030
	                ICdmApplicationConfiguration cdmAppController = config.getCdmAppController();
1031
	                if(cdmAppController == null){
1032
	                    cdmAppController = this;
1033
	                }
1034
	                specimenTypeDesignationstatus = (SpecimenTypeDesignationStatus) cdmAppController.getTermService().find(specimenTypeDesignationstatus.getUuid());
1035
	                //Designation
1036
	                TaxonNameBase<?,?> name = taxon.getName();
1037
	                SpecimenTypeDesignation designation = SpecimenTypeDesignation.NewInstance();
1038

    
1039
	                designation.setTypeStatus(specimenTypeDesignationstatus);
1040
	                designation.setTypeSpecimen(state.getDerivedUnitBase());
1041
	                name.addTypeDesignation(designation, true);
1042
	            }
1043
	        }
1044

    
1045
	        for (String[] fullReference : state.getDataHolder().getReferenceList()) {
1046

    
1047

    
1048
	            String strReference=fullReference[0];
1049
	            String citationDetail = fullReference[1];
1050
	            String citationURL = fullReference[2];
1051
	            List<Reference> references = getReferenceService().listByTitle(Reference.class, "strReference", MatchMode.EXACT, null, null, null, null, null);
1052

    
1053
	            if (!references.isEmpty()){
1054
	                Reference reference = null;
1055
	                for (Reference refe: references) {
1056
	                    if (refe.getTitleCache().equalsIgnoreCase(strReference)) {
1057
	                        reference =refe;
1058
	                        break;
1059
	                    }
1060
	                }
1061
	                if (reference ==null){
1062
	                    reference = ReferenceFactory.newGeneric();
1063
	                    reference.setTitleCache(strReference, true);
1064
	                    save(reference, state);
1065
	                }
1066
	                determinationEvent.addReference(reference);
1067
	            }
1068
	        }
1069
	        save(state.getDerivedUnitBase(), state);
1070

    
1071
	        if (config.isAddIndividualsAssociationsSuchAsSpecimenAndObservations() && preferredFlag) {
1072
	            //do not add IndividualsAssociation to non-preferred taxa
1073
	            if(DEBUG){
1074
	                logger.info("isDoCreateIndividualsAssociations");
1075
	            }
1076

    
1077
	            makeIndividualsAssociation(state, taxon, determinationEvent);
1078

    
1079
	            save(state.getDerivedUnitBase(), state);
1080
	        }
1081
	    }
1082

    
1083
	    /**
1084
	     * create and link each association (specimen, observation..) to the accepted taxon
1085
	     * @param state : the ABCD import state
1086
	     * @param taxon: the current Taxon
1087
	     * @param determinationEvent:the determinationevent
1088
	     */
1089
	    protected void makeIndividualsAssociation(STATE state, Taxon taxon, DeterminationEvent determinationEvent) {
1090
	        SpecimenImportConfiguratorBase config = state.getConfig();
1091
	        SpecimenUserInteraction sui = config.getSpecimenUserInteraction();
1092

    
1093
	        if (DEBUG) {
1094
	            logger.info("MAKE INDIVIDUALS ASSOCIATION");
1095
	        }
1096

    
1097
	        TaxonDescription taxonDescription = null;
1098
	        Set<TaxonDescription> descriptions= taxon.getDescriptions();
1099
//	        if (((Abcd206ImportConfigurator) state.getConfig()).isInteractWithUser()){
1100
//	            if(!state.isDescriptionGroupSet()){
1101
//	                taxonDescription = sui.askForDescriptionGroup(descriptions);
1102
//	                state.setDescriptionGroup(taxonDescription);
1103
//	                state.setDescriptionGroupSet(true);
1104
//	            }else{
1105
//	                taxonDescription=state.getDescriptionGroup();
1106
//	            }
1107
//	        } else {
1108
	            for (TaxonDescription description : descriptions){
1109
	                Set<IdentifiableSource> sources =  new HashSet<>();
1110
	                sources.addAll(description.getTaxon().getSources());
1111
	                sources.addAll(description.getSources());
1112
	                for (IdentifiableSource source:sources){
1113
	                    if(state.getRef().equals(source.getCitation())) {
1114
	                        taxonDescription = description;
1115
	                    }
1116
	                }
1117
	            }
1118
	      //  }
1119
	        if (taxonDescription == null){
1120
	            taxonDescription = TaxonDescription.NewInstance(taxon, false);
1121
	            if(sourceNotLinkedToElement(taxonDescription,state.getRef(),null)) {
1122
	                taxonDescription.addSource(OriginalSourceType.Import, null, null, state.getRef(), null);
1123
	            }
1124
	            state.setDescriptionGroup(taxonDescription);
1125
	            taxon.addDescription(taxonDescription);
1126
	        }
1127

    
1128
	        //PREPARE REFERENCE QUESTIONS
1129

    
1130
	        Map<String,OriginalSourceBase<?>> sourceMap = new HashMap<String, OriginalSourceBase<?>>();
1131

    
1132
	        List<IdentifiableSource> issTmp = new ArrayList<>();//getCommonService().list(IdentifiableSource.class, null, null, null, null);
1133
	        List<DescriptionElementSource> issTmp2 = new ArrayList<>();//getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1134

    
1135
	        Set<OriginalSourceBase> osbSet = new HashSet<OriginalSourceBase>();
1136
	        if(issTmp2!=null) {
1137
	            osbSet.addAll(issTmp2);
1138
	        }
1139
	        if(issTmp!=null) {
1140
	            osbSet.addAll(issTmp);
1141
	        }
1142

    
1143

    
1144
	        addToSourceMap(sourceMap, osbSet);
1145

    
1146
//	        if (((Abcd206ImportConfigurator) state.getConfig()).isInteractWithUser()){
1147
//	            List<OriginalSourceBase<?>> res = null;
1148
//	            if(!state.isDescriptionSourcesSet()){
1149
//	                res = sui.askForSource(sourceMap, "the description group ("+taxon+")",
1150
//	                        "The current reference is "+state.getRef().getTitleCache(),getReferenceService(), state.getDataHolder().docSources);
1151
//	                state.setDescriptionRefs(res);
1152
//	                state.setDescriptionSourcesSet(true);
1153
//	            }
1154
//	            else{
1155
//	                res=state.getDescriptionRefs();
1156
//	            }
1157
//	            if(res !=null) {
1158
//	                for (OriginalSourceBase<?> sour:res){
1159
//	                    if(sour.isInstanceOf(IdentifiableSource.class)){
1160
//	                        try {
1161
//	                            if(sourceNotLinkedToElement(taxonDescription,sour)) {
1162
//	                                taxonDescription.addSource((IdentifiableSource)sour.clone());
1163
//	                            }
1164
//	                        } catch (CloneNotSupportedException e) {
1165
//	                            logger.warn("no cloning?");
1166
//	                        }
1167
//	                    }else{
1168
//	                        if(sourceNotLinkedToElement(taxonDescription,sour)) {
1169
//	                            taxonDescription.addSource(OriginalSourceType.Import,null, null, sour.getCitation(),sour.getCitationMicroReference());
1170
//	                        }
1171
//	                    }
1172
//	                }
1173
//	            }
1174
//	        }
1175
//	        else {
1176
	            if(sourceNotLinkedToElement(taxonDescription,state.getRef(),null)) {
1177
	                taxonDescription.addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1178
	            }
1179
//	        }
1180
	        state.setDescriptionGroup(taxonDescription);
1181

    
1182
	        IndividualsAssociation indAssociation = IndividualsAssociation.NewInstance();
1183
	        Feature feature = makeFeature(state.getDerivedUnitBase());
1184
	        indAssociation.setAssociatedSpecimenOrObservation(state.getDerivedUnitBase());
1185
	        indAssociation.setFeature(feature);
1186
//
1187
//	        if (((Abcd206ImportConfigurator) state.getConfig()).isInteractWithUser()){
1188
//	            sourceMap = new HashMap<String, OriginalSourceBase<?>>();
1189
//
1190
//	            issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
1191
//	            issTmp2 = getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1192
//
1193
//	            osbSet = new HashSet<OriginalSourceBase>();
1194
//	            if(issTmp2!=null) {
1195
//	                osbSet.addAll(issTmp2);
1196
//	            }
1197
//	            if(issTmp!=null) {
1198
//	                osbSet.addAll(issTmp);
1199
//	            }
1200
//
1201
//
1202
//	            addToSourceMap(sourceMap, osbSet);
1203
//
1204
//	            List<OriginalSourceBase<?>> sources =null;
1205
//	            if(!state.isAssociationSourcesSet()) {
1206
//	                sources = sui.askForSource(sourceMap,  "descriptive element (association) ",taxon.toString(),
1207
//	                        getReferenceService(),state.getDataHolder().getDocSources());
1208
//	                state.setAssociationRefs(sources);
1209
//	                state.setAssociationSourcesSet(true);
1210
//	            }
1211
//	            else{
1212
//	                sources=state.getAssociationRefs();
1213
//	            }
1214
//	            if(sources !=null) {
1215
//	                for (OriginalSourceBase<?> source: sources) {
1216
//	                    if(source !=null) {
1217
//	                        if(source.isInstanceOf(DescriptionElementSource.class)){
1218
//	                            try {
1219
//	                                if(sourceNotLinkedToElement(indAssociation,source)) {
1220
//	                                    indAssociation.addSource((DescriptionElementSource)source.clone());
1221
//	                                }
1222
//	                            } catch (CloneNotSupportedException e) {
1223
//	                                logger.warn("clone forbidden?");
1224
//	                            }
1225
//	                        }else{
1226
//	                            if(sourceNotLinkedToElement(indAssociation,source)) {
1227
//	                                indAssociation.addSource(OriginalSourceType.Import,null, null, source.getCitation(),source.getCitationMicroReference());
1228
//	                            }
1229
//	                            try {
1230
//	                                if(sourceNotLinkedToElement(state.getDerivedUnitBase(), source)) {
1231
//	                                    state.getDerivedUnitBase().addSource((IdentifiableSource) source.clone());
1232
//	                                }
1233
//	                            } catch (CloneNotSupportedException e) {
1234
//	                                // TODO Auto-generated catch block
1235
//	                                e.printStackTrace();
1236
//	                            }
1237
//	                        }
1238
//
1239
//	                    }
1240
//	                }
1241
//	            }
1242
//	        }else {
1243
	            if(sourceNotLinkedToElement(indAssociation,state.getRef(),null)) {
1244
	                indAssociation.addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1245
	            }
1246
	            if(sourceNotLinkedToElement(state.getDerivedUnitBase(), state.getRef(),null)) {
1247
	                state.getDerivedUnitBase().addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1248
	            }
1249
	            for (Reference citation : determinationEvent.getReferences()) {
1250
	                if(sourceNotLinkedToElement(indAssociation,citation,null))
1251
	                {
1252
	                    indAssociation.addSource(DescriptionElementSource.NewInstance(OriginalSourceType.Import, null, null, citation, null));
1253
	                }
1254
	                if(sourceNotLinkedToElement(state.getDerivedUnitBase(), state.getRef(),null)) {
1255
	                    state.getDerivedUnitBase().addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1256
	                }
1257
	            }
1258
	   //     }
1259

    
1260
	        taxonDescription.addElement(indAssociation);
1261

    
1262
	        save(taxonDescription, state);
1263
	        save(taxon, state);
1264
	        state.getReport().addDerivate(state.getDerivedUnitBase(), config);
1265
	        state.getReport().addIndividualAssociation(taxon, state.getDataHolder().getUnitID(), state.getDerivedUnitBase());
1266
	    }
1267

    
1268
	    /**
1269
	     * @param derivedUnitBase2
1270
	     * @param ref2
1271
	     * @param object
1272
	     * @return
1273
	     */
1274
	    private boolean sourceNotLinkedToElement(DerivedUnit derivedUnitBase2, Reference b, String d) {
1275
	        Set<IdentifiableSource> linkedSources = derivedUnitBase2.getSources();
1276
	        for (IdentifiableSource is:linkedSources){
1277
	            Reference a = is.getCitation();
1278
	            String c = is.getCitationMicroReference();
1279

    
1280
	            boolean refMatch=false;
1281
	            boolean microMatch=false;
1282

    
1283
	            try{
1284
	                if (a==null && b==null) {
1285
	                    refMatch=true;
1286
	                }
1287
	                if (a!=null && b!=null) {
1288
	                    if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1289
	                        refMatch=true;
1290
	                    }
1291
	                }
1292
	            }catch(Exception e){}
1293

    
1294

    
1295
	            try{
1296
	                if (c==null && d==null) {
1297
	                    microMatch=true;
1298
	                }
1299
	                if(c!=null && d!=null) {
1300
	                    if(c.equalsIgnoreCase(d)) {
1301
	                        microMatch=true;
1302
	                    }
1303
	                }
1304
	            }
1305
	            catch(Exception e){}
1306

    
1307
	            if (microMatch && refMatch) {
1308
	                return false;
1309
	            }
1310

    
1311

    
1312
	        }
1313
	        return true;
1314
	    }
1315

    
1316
	    private <T extends OriginalSourceBase<?>> boolean  sourceNotLinkedToElement(ISourceable<T> sourcable, Reference reference, String microReference) {
1317
	        Set<T> linkedSources = sourcable.getSources();
1318
	        for (T is:linkedSources){
1319
	            Reference unitReference = is.getCitation();
1320
	            String unitMicroReference = is.getCitationMicroReference();
1321

    
1322
	            boolean refMatch=false;
1323
	            boolean microMatch=false;
1324

    
1325
	            try{
1326
	                if (unitReference==null && reference==null) {
1327
	                    refMatch=true;
1328
	                }
1329
	                if (unitReference!=null && reference!=null) {
1330
	                    if (unitReference.getTitleCache().equalsIgnoreCase(reference.getTitleCache())) {
1331
	                        refMatch=true;
1332
	                    }
1333
	                }
1334
	            }catch(Exception e){}
1335

    
1336
	            try{
1337
	                if (unitMicroReference==null && microReference==null) {
1338
	                    microMatch=true;
1339
	                }
1340
	                if(unitMicroReference!=null && microReference!=null) {
1341
	                    if(unitMicroReference.equalsIgnoreCase(microReference)) {
1342
	                        microMatch=true;
1343
	                    }
1344
	                }
1345
	            }
1346
	            catch(Exception e){}
1347

    
1348
	            if (microMatch && refMatch) {
1349
	                return false;
1350
	            }
1351
	        }
1352
	        return true;
1353
	    }
1354

    
1355
	    /**
1356
	     * look for the Feature object (FieldObs, Specimen,...)
1357
	     * @param unit : a specimen or obersvation base
1358
	     * @return the corresponding Feature
1359
	     */
1360
	    private Feature makeFeature(SpecimenOrObservationBase<?> unit) {
1361
	        SpecimenOrObservationType type = unit.getRecordBasis();
1362

    
1363

    
1364

    
1365
	        if (type.isFeatureObservation()){
1366
	            return Feature.OBSERVATION();
1367
	        }else if (type.isFeatureSpecimen()){
1368
	            return Feature.SPECIMEN();
1369
	        }else if (type == SpecimenOrObservationType.DerivedUnit){
1370
	            return Feature.OBSERVATION();
1371
	            //            return getFeature("Specimen or observation");
1372
	        }else{
1373
	            String message = "Unhandled record basis '%s' for defining individuals association feature type. Use default.";
1374
	            logger.warn(String.format(message, type.getMessage()));
1375
	            return Feature.OBSERVATION();
1376
	            //            return getFeature("Specimen or observation");
1377

    
1378
	        }
1379
	    }
1380

    
1381

    
1382
	    /**
1383
	     * @param sourceMap
1384
	     * @param osbSet
1385
	     */
1386
	    protected void addToSourceMap(Map<String, OriginalSourceBase<?>> sourceMap, Set<OriginalSourceBase> osbSet) {
1387
	        for( OriginalSourceBase<?> osb:osbSet) {
1388
	            if(osb.getCitation()!=null && osb.getCitationMicroReference() !=null  && !osb.getCitationMicroReference().isEmpty()) {
1389
	                try{
1390
	                    sourceMap.put(osb.getCitation().getTitleCache()+ "---"+osb.getCitationMicroReference(),osb);
1391
	                }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1392
	            } else if(osb.getCitation()!=null){
1393
	                try{
1394
	                    sourceMap.put(osb.getCitation().getTitleCache(),osb);
1395
	                }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1396
	            }
1397
	        }
1398
	    }
1399

    
1400
}
(2-2/7)