Project

General

Profile

Download (64.6 KB) Statistics
| Branch: | Tag: | Revision:
1
/**
2
* Copyright (C) 2007 EDIT
3
* European Distributed Institute of Taxonomy
4
* http://www.e-taxonomy.eu
5
*
6
* The contents of this file are subject to the Mozilla Public License Version 1.1
7
* See LICENSE.TXT at the top of this package for the full license terms.
8
*/
9

    
10
package eu.etaxonomy.cdm.io.specimen;
11

    
12

    
13
import java.util.ArrayList;
14
import java.util.HashMap;
15
import java.util.HashSet;
16
import java.util.List;
17
import java.util.Map;
18
import java.util.Set;
19
import java.util.UUID;
20

    
21
import org.apache.log4j.Logger;
22

    
23
import eu.etaxonomy.cdm.api.application.ICdmRepository;
24
import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
25
import eu.etaxonomy.cdm.api.service.config.FindOccurrencesConfigurator;
26
import eu.etaxonomy.cdm.api.service.pager.Pager;
27
import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
28
import eu.etaxonomy.cdm.io.common.CdmImportBase;
29
import eu.etaxonomy.cdm.io.common.IImportConfigurator;
30
import eu.etaxonomy.cdm.io.specimen.abcd206.in.Identification;
31
import eu.etaxonomy.cdm.io.specimen.abcd206.in.SpecimenImportReport;
32
import eu.etaxonomy.cdm.model.agent.AgentBase;
33
import eu.etaxonomy.cdm.model.agent.Institution;
34
import eu.etaxonomy.cdm.model.agent.Person;
35
import eu.etaxonomy.cdm.model.agent.Team;
36
import eu.etaxonomy.cdm.model.common.CdmBase;
37
import eu.etaxonomy.cdm.model.common.ISourceable;
38
import eu.etaxonomy.cdm.model.common.IdentifiableSource;
39
import eu.etaxonomy.cdm.model.common.LanguageString;
40
import eu.etaxonomy.cdm.model.common.OriginalSourceBase;
41
import eu.etaxonomy.cdm.model.common.OriginalSourceType;
42
import eu.etaxonomy.cdm.model.description.DescriptionBase;
43
import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
44
import eu.etaxonomy.cdm.model.description.Feature;
45
import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
46
import eu.etaxonomy.cdm.model.description.TaxonDescription;
47
import eu.etaxonomy.cdm.model.name.INonViralName;
48
import eu.etaxonomy.cdm.model.name.ITaxonNameBase;
49
import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
50
import eu.etaxonomy.cdm.model.name.Rank;
51
import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
52
import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
53
import eu.etaxonomy.cdm.model.name.TaxonName;
54
import eu.etaxonomy.cdm.model.name.TaxonNameFactory;
55
import eu.etaxonomy.cdm.model.occurrence.Collection;
56
import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
57
import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
58
import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
59
import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationType;
60
import eu.etaxonomy.cdm.model.reference.Reference;
61
import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
62
import eu.etaxonomy.cdm.model.taxon.Classification;
63
import eu.etaxonomy.cdm.model.taxon.Synonym;
64
import eu.etaxonomy.cdm.model.taxon.Taxon;
65
import eu.etaxonomy.cdm.model.taxon.TaxonBase;
66
import eu.etaxonomy.cdm.model.taxon.TaxonNode;
67
import eu.etaxonomy.cdm.persistence.query.MatchMode;
68
import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
69
import eu.etaxonomy.cdm.strategy.parser.ParserProblem;
70
import eu.etaxonomy.cdm.strategy.parser.TimePeriodParser;
71

    
72

    
73
/**
74
 * @author p.kelbert
75
 * @created 20.10.2008
76
 */
77
public abstract class SpecimenImportBase<CONFIG extends IImportConfigurator, STATE extends SpecimenImportStateBase>
78
        extends CdmImportBase<CONFIG, STATE> {
79

    
80
    private static final long serialVersionUID = 4423065367998125678L;
81
    private static final Logger logger = Logger.getLogger(SpecimenImportBase.class);
82

    
83
	protected static final UUID SPECIMEN_SCAN_TERM = UUID.fromString("acda15be-c0e2-4ea8-8783-b9b0c4ad7f03");
84

    
85
	private static final String COLON = ":";
86

    
87

    
88
	@Override
89
    protected abstract void doInvoke(STATE state);
90

    
91
	/**
92
	 * Handle a single unit
93
	 * @param state
94
	 * @param item
95
	 */
96
	protected abstract void handleSingleUnit(STATE state, Object item) ;
97

    
98

    
99

    
100
	protected TaxonName getOrCreateTaxonName(String scientificName, Rank rank, boolean preferredFlag, STATE state, int unitIndexInAbcdFile){
101
	    TaxonName taxonName = null;
102
        SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
103

    
104
        //check atomised name data for rank
105
        //new name will be created
106
        ITaxonNameBase atomisedTaxonName = null;
107
        if (rank==null && unitIndexInAbcdFile>=0 && ((state.getDataHolder().getAtomisedIdentificationList() != null && !state.getDataHolder().getAtomisedIdentificationList().isEmpty())|| state.getDataHolder().getAtomisedIdentificationList().size() > 0)) {
108
            atomisedTaxonName = setTaxonNameByType(state.getDataHolder().getAtomisedIdentificationList().get(unitIndexInAbcdFile), scientificName, state);
109
            if(atomisedTaxonName!=null){
110
                rank = atomisedTaxonName.getRank();
111
            }
112
        }
113
        if(config.isReuseExistingTaxaWhenPossible()){
114
            ITaxonNameBase parsedName = atomisedTaxonName;
115
            if(parsedName==null){
116

    
117
                parsedName = parseScientificName(scientificName, state, state.getReport(), rank);
118

    
119
            }
120
            atomisedTaxonName = parsedName;
121
            if(config.isIgnoreAuthorship() && parsedName!=null){// && preferredFlag){
122
                // do not ignore authorship for non-preferred names because they need
123
                // to be created for the determination history
124
                String nameCache = TaxonName.castAndDeproxy(parsedName).getNameCache();
125
                List<TaxonName> names = getNameService().findNamesByNameCache(nameCache, MatchMode.EXACT, null);
126
                if (!names.isEmpty()){
127
                     taxonName = getBestMatchingName(scientificName, new ArrayList<>(names), state);
128
                }
129
                if (taxonName == null && !names.isEmpty()){
130
                    taxonName = names.get(0);
131
                }
132

    
133
            } else {
134
                //search for existing names
135
                List<TaxonName> names = getNameService().listByTitle(TaxonName.class, scientificName, MatchMode.EXACT, null, null, null, null, null);
136
                taxonName = getBestMatchingName(scientificName, names, state);
137
                //still nothing found -> try with the atomised name full title cache
138
                if(taxonName==null && atomisedTaxonName!=null){
139
                    names = getNameService().listByTitle(TaxonName.class, atomisedTaxonName.getFullTitleCache(), MatchMode.EXACT, null, null, null, null, null);
140
                    taxonName = getBestMatchingName(atomisedTaxonName.getTitleCache(), names, state);
141
                    //still nothing found -> try with the atomised name title cache
142
                    if(taxonName==null){
143
                        names = getNameService().listByTitle(TaxonName.class, atomisedTaxonName.getTitleCache(), MatchMode.EXACT, null, null, null, null, null);
144
                        taxonName = getBestMatchingName(atomisedTaxonName.getTitleCache(), names, state);
145
                    }
146
                }
147

    
148
            }
149

    
150
        }
151

    
152
        if(taxonName==null && atomisedTaxonName!=null){
153
            taxonName = (TaxonName) atomisedTaxonName;
154
            state.getReport().addName(taxonName);
155
            logger.info("Created new taxon name "+taxonName);
156
            if(taxonName.hasProblem()){
157
                state.getReport().addInfoMessage(String.format("Created %s with parsing problems", taxonName));
158
            }
159
            if(!atomisedTaxonName.getTitleCache().equals(scientificName)){
160
                state.getReport().addInfoMessage(String.format("Taxon %s was parsed as %s", scientificName, atomisedTaxonName.getTitleCache()));
161
            }
162
        }
163
        else if(taxonName==null){
164
            //create new taxon name
165

    
166
            if (state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNAFP)){
167
                taxonName = TaxonNameFactory.NewBotanicalInstance(rank);
168
            }else if (state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICZN)){
169
                taxonName = TaxonNameFactory.NewZoologicalInstance(rank);
170
            }else{
171
                taxonName = TaxonNameFactory.NewNonViralInstance(rank);
172
            }
173
            taxonName.setFullTitleCache(scientificName,true);
174
            taxonName.setTitleCache(scientificName, true);
175
            state.getReport().addName(taxonName);
176
            logger.info("Created new taxon name "+taxonName);
177
        }
178
        save(taxonName, state);
179
        return taxonName;
180
    }
181

    
182
	 protected TaxonName getBestMatchingName(String scientificName, java.util.Collection<TaxonName> names, STATE state){
183
	        Set<TaxonName> namesWithAcceptedTaxa = new HashSet<>();
184
	        List<TaxonName> namesWithAcceptedTaxaInClassification = new ArrayList<>();
185
	        for (TaxonName name : names) {
186
	            if(!name.getTaxa().isEmpty()){
187
	                Set<Taxon> taxa = name.getTaxa();
188
	                for (Taxon taxon:taxa){
189
	                    if (!taxon.getTaxonNodes().isEmpty()){
190
	                        //use only taxa included in a classification
191
	                        for (TaxonNode node:taxon.getTaxonNodes()){
192
	                            if (state.getClassification() != null && node.getClassification().equals(state.getClassification())){
193
	                                namesWithAcceptedTaxaInClassification.add(name);
194
	                            }else {
195
	                                namesWithAcceptedTaxa.add(name);
196
	                            }
197
	                        }
198

    
199
	                    }
200
	                }
201

    
202
	            }
203
	        }
204
	        String message = String.format("More than one taxon name was found for %s, maybe in other classifications!", scientificName);
205
	        //check for names with accepted taxa in classification
206
	        if(namesWithAcceptedTaxaInClassification.size()>0){
207
                if(namesWithAcceptedTaxaInClassification.size()>1){
208

    
209
                    state.getReport().addInfoMessage(message);
210
                    logger.warn(message);
211
                    return null;
212
                }
213
                return namesWithAcceptedTaxaInClassification.iterator().next();
214
            }
215
	      //check for any names with accepted taxa
216
	        if(namesWithAcceptedTaxa.size()>0){
217
	            if(namesWithAcceptedTaxa.size()>1){
218

    
219
	                state.getReport().addInfoMessage(message);
220
	                logger.warn(message);
221
	                return null;
222
	            }
223
	            return namesWithAcceptedTaxa.iterator().next();
224
	        }
225
	        //no names with accepted taxa found -> check accepted taxa of synonyms
226
	        List<Taxon> taxaFromSynonyms = new ArrayList<>();
227
	        for (TaxonName name : names) {
228
	            Set<TaxonBase> taxonBases = name.getTaxonBases();
229
	            for (TaxonBase taxonBase : taxonBases) {
230
	                if(taxonBase.isInstanceOf(Synonym.class)){
231
	                    Synonym synonym = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
232
	                    taxaFromSynonyms.add(synonym.getAcceptedTaxon());
233
	                }
234
	            }
235
	        }
236
	        if(taxaFromSynonyms.size()>0){
237
	            if(taxaFromSynonyms.size()>1){
238
	                state.getReport().addInfoMessage(message);
239
	                logger.warn(message);
240
	                return null;
241
	            }
242
	            return taxaFromSynonyms.iterator().next().getName();
243
	        }
244
	        //no accepted and no synonyms -> return one of the names and create a new taxon
245
	        if (names.isEmpty()){
246
	            return null;
247
	        }else{
248
	            return names.iterator().next();
249
	        }
250
	    }
251
	 /**
252
	     * Parse automatically the scientific name
253
	     * @param scientificName the scientific name to parse
254
	     * @param state the current import state
255
	     * @param report the import report
256
	     * @return a parsed name
257
	     */
258

    
259
	    protected ITaxonNameBase parseScientificName(String scientificName, STATE state, SpecimenImportReport report, Rank rank) {
260

    
261
	        NonViralNameParserImpl nvnpi = NonViralNameParserImpl.NewInstance();
262
	        ITaxonNameBase taxonName = null;
263
	        boolean problem = false;
264

    
265
	        if (logger.isDebugEnabled()){
266
	            logger.debug("parseScientificName " + state.getDataHolder().getNomenclatureCode().toString());
267
	        }
268

    
269
	        if (state.getDataHolder().getNomenclatureCode().toString().equals("Zoological") || state.getDataHolder().getNomenclatureCode().toString().contains("ICZN")) {
270
	            taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICZN, rank);
271
	            if (taxonName.hasProblem()) {
272
	                problem = true;
273
	            }
274
	        }
275
	        else if (state.getDataHolder().getNomenclatureCode().toString().equals("Botanical") || state.getDataHolder().getNomenclatureCode().toString().contains("ICBN")) {
276
	            taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNAFP, rank);
277
	            if (taxonName.hasProblem()) {
278
	                problem = true;
279
	            }
280
	        }
281
	        else if (state.getDataHolder().getNomenclatureCode().toString().equals("Bacterial") || state.getDataHolder().getNomenclatureCode().toString().contains("ICBN")) {
282
	            taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNB, rank);
283
	            if (taxonName.hasProblem()) {
284
	                problem = true;
285
	            }
286
	        }
287
	        else if (state.getDataHolder().getNomenclatureCode().toString().equals("Cultivar") || state.getDataHolder().getNomenclatureCode().toString().contains("ICNCP")) {
288
	            taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNCP, rank);
289
	            if (taxonName.hasProblem()) {
290
	                problem = true;
291
	            }
292
	        }
293
	        if (problem) {
294
	            String message = String.format("Parsing problems for %s", scientificName);
295
	            if(taxonName!=null){
296
	                for (ParserProblem parserProblem : taxonName.getParsingProblems()) {
297
	                    message += "\n\t- "+parserProblem;
298
	                }
299
	            }
300
	            report.addInfoMessage(message);
301
	            logger.info(message);
302
	        }
303
	        return taxonName;
304

    
305
	    }
306

    
307
	    /**
308
	     * Create the name without automatic parsing, either because it failed, or because the user deactivated it.
309
	     * The name is built upon the ABCD fields
310
	     * @param atomisedMap : the ABCD atomised fields
311
	     * @param fullName : the full scientific name
312
	     * @param state
313
	     * @return the corresponding Botanical or Zoological or... name
314
	     */
315
	    protected TaxonName setTaxonNameByType(
316
	            HashMap<String, String> atomisedMap, String fullName, STATE state) {
317
	        boolean problem = false;
318
	        if (logger.isDebugEnabled()){
319
	            logger.debug("settaxonnamebytype " + state.getDataHolder().getNomenclatureCode().toString());
320
	        }
321

    
322
	        if (state.getDataHolder().getNomenclatureCode().equals("Zoological") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICZN.getUuid())) {
323
	            TaxonName taxonName = TaxonNameFactory.NewZoologicalInstance(null);
324
	            taxonName.setFullTitleCache(fullName, true);
325
	            taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
326
	            taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
327
	            taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap,"SpeciesEpithet")));
328
	            taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap,"SubspeciesEpithet")));
329

    
330
	            if (taxonName.getGenusOrUninomial() != null){
331
	                taxonName.setRank(Rank.GENUS());
332
	            }
333

    
334
	            if (taxonName.getInfraGenericEpithet() != null){
335
	                taxonName.setRank(Rank.SUBGENUS());
336
	            }
337

    
338
	            if (taxonName.getSpecificEpithet() != null){
339
	                taxonName.setRank(Rank.SPECIES());
340
	            }
341

    
342
	            if (taxonName.getInfraSpecificEpithet() != null){
343
	                taxonName.setRank(Rank.SUBSPECIES());
344
	            }
345

    
346
	            Team team = null;
347
	            if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
348
	                team = Team.NewInstance();
349
	                team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
350
	            }
351
	            else {
352
	                if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
353
	                    team = Team.NewInstance();
354
	                    team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
355
	                }
356
	            }
357
	            if (team != null) {
358
	                taxonName.setBasionymAuthorship(team);
359
	            }
360
	            else {
361
	                if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
362
	                    taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
363
	                }
364
	                else if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
365
	                    taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamAndYear"));
366
	                }
367
	            }
368
	            if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
369
	                team = Team.NewInstance();
370
	                team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
371
	                taxonName.setCombinationAuthorship(team);
372
	            }
373
	            if (taxonName.hasProblem()) {
374
	                logger.info("pb ICZN");
375
	                problem = true;
376
	            }
377
	            else {
378
	                return taxonName;
379
	            }
380
	        }
381
	        else if (state.getDataHolder().getNomenclatureCode().equals("Botanical") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNAFP.getUuid())) {
382
	            TaxonName taxonName = (TaxonName)parseScientificName(fullName, state, state.getReport(), null);
383
	            if (taxonName != null){
384
	                return taxonName;
385
	            }
386
	            else{
387
	                taxonName = TaxonNameFactory.NewBotanicalInstance(null);
388
	            }
389
	            taxonName.setFullTitleCache(fullName, true);
390
	            taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
391
	            taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "FirstEpithet")));
392
	            taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "InfraSpeEpithet")));
393
	            try {
394
	                taxonName.setRank(Rank.getRankByName(getFromMap(atomisedMap, "Rank")));
395
	            } catch (Exception e) {
396
	                if (taxonName.getInfraSpecificEpithet() != null){
397
	                    taxonName.setRank(Rank.SUBSPECIES());
398
	                }
399
	                else if (taxonName.getSpecificEpithet() != null){
400
	                    taxonName.setRank(Rank.SPECIES());
401
	                }
402
	                else if (taxonName.getInfraGenericEpithet() != null){
403
	                    taxonName.setRank(Rank.SUBGENUS());
404
	                }
405
	                else if (taxonName.getGenusOrUninomial() != null){
406
	                    taxonName.setRank(Rank.GENUS());
407
	                }
408
	            }
409
	            Team team = null;
410
	            if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
411
	                team = Team.NewInstance();
412
	                team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
413
	                taxonName.setBasionymAuthorship(team);
414
	            }
415
	            if (getFromMap(atomisedMap, "AuthorTeam") != null) {
416
	                team = Team.NewInstance();
417
	                team.setTitleCache(getFromMap(atomisedMap, "AuthorTeam"), true);
418
	                taxonName.setCombinationAuthorship(team);
419
	            }
420
	            if (team == null) {
421
	                if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
422
	                    taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
423
	                }
424
	                else if (getFromMap(atomisedMap, "AuthorTeam") != null) {
425
	                    taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeam"));
426
	                }
427
	            }
428
	            if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
429
	                team = Team.NewInstance();
430
	                team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
431
	                taxonName.setCombinationAuthorship(team);
432
	            }
433
	            if (taxonName.hasProblem()) {
434
	                logger.info("pb ICBN");
435
	                problem = true;
436
	            }
437
	            else {
438
	                return taxonName;
439
	            }
440
	        }
441
	        else if (state.getDataHolder().getNomenclatureCode().equals("Bacterial") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNB.getUuid())) {
442
	            TaxonName taxonName = TaxonNameFactory.NewBacterialInstance(null);
443
	            taxonName.setFullTitleCache(fullName, true);
444
	            taxonName.setGenusOrUninomial(getFromMap(atomisedMap, "Genus"));
445
	            taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
446
	            taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "Species")));
447
	            taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "SubspeciesEpithet")));
448

    
449
	            if (taxonName.getGenusOrUninomial() != null){
450
	                taxonName.setRank(Rank.GENUS());
451
	            }
452
	            else if (taxonName.getInfraGenericEpithet() != null){
453
	                taxonName.setRank(Rank.SUBGENUS());
454
	            }
455
	            else if (taxonName.getSpecificEpithet() != null){
456
	                taxonName.setRank(Rank.SPECIES());
457
	            }
458
	            else if (taxonName.getInfraSpecificEpithet() != null){
459
	                taxonName.setRank(Rank.SUBSPECIES());
460
	            }
461

    
462
	            if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
463
	                Team team = Team.NewInstance();
464
	                team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
465
	                taxonName.setCombinationAuthorship(team);
466
	            }
467
	            if (getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear") != null) {
468
	                Team team = Team.NewInstance();
469
	                team.setTitleCache(getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear"), true);
470
	                taxonName.setBasionymAuthorship(team);
471
	            }
472
	            if (taxonName.hasProblem()) {
473
	                logger.info("pb ICNB");
474
	                problem = true;
475
	            }
476
	            else {
477
	                return taxonName;
478
	            }
479
	        }
480
	        else if (state.getDataHolder().getNomenclatureCode().equals("Cultivar")) {
481
	            TaxonName taxonName = TaxonNameFactory.NewCultivarInstance(null);
482

    
483
	            if (taxonName.hasProblem()) {
484
	                logger.info("pb ICNCP");
485
	                problem = true;
486
	            }
487
	            else {
488
	                return taxonName;
489
	            }
490
	            return taxonName;
491
	        }
492

    
493
	        if (problem) {
494
	            logger.info("Problem im setTaxonNameByType ");
495
	            TaxonName taxonName = TaxonNameFactory.NewNonViralInstance(null);
496
	            taxonName.setFullTitleCache(fullName, true);
497
	            return taxonName;
498
	        }
499
	        TaxonName tn = TaxonNameFactory.NewNonViralInstance(null);
500
	        return tn;
501
	    }
502

    
503
	    /**
504
	     * Get a formated string from a hashmap
505
	     * @param atomisedMap
506
	     * @param key
507
	     * @return
508
	     */
509
	    private String getFromMap(HashMap<String, String> atomisedMap, String key) {
510
	        String value = null;
511
	        if (atomisedMap.containsKey(key)) {
512
	            value = atomisedMap.get(key);
513
	        }
514

    
515
	        try {
516
	            if (value != null && key.matches(".*Year.*")) {
517
	                value = value.trim();
518
	                if (value.matches("[a-z A-Z ]*[0-9]{4}$")) {
519
	                    String tmp = value.split("[0-9]{4}$")[0];
520
	                    int year = Integer.parseInt(value.split(tmp)[1]);
521
	                    if (year >= 1752) {
522
	                        value = tmp;
523
	                    }
524
	                    else {
525
	                        value = null;
526
	                    }
527
	                }
528
	                else {
529
	                    value = null;
530
	                }
531
	            }
532
	        }
533
	        catch (Exception e) {
534
	            value = null;
535
	        }
536
	        return value;
537
	    }
538

    
539
	    /**
540
	     * Very fast and dirty implementation to allow handling of transient objects as described in
541
	     * https://dev.e-taxonomy.eu/trac/ticket/3726
542
	     *
543
	     * Not yet complete.
544
	     *
545
	     * @param cdmBase
546
	     * @param state
547
	     */
548
	    protected UUID save(CdmBase cdmBase, SpecimenImportStateBase state) {
549
	        ICdmRepository cdmRepository = state.getConfig().getCdmAppController();
550
	        if (cdmRepository == null){
551
	            cdmRepository = this;
552
	        }
553

    
554
	        if (cdmBase.isInstanceOf(LanguageString.class)){
555
	            return cdmRepository.getTermService().saveLanguageData(CdmBase.deproxy(cdmBase, LanguageString.class));
556
	        }else if (cdmBase.isInstanceOf(SpecimenOrObservationBase.class)){
557
	            return cdmRepository.getOccurrenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, SpecimenOrObservationBase.class));
558
	        }else if (cdmBase.isInstanceOf(Reference.class)){
559
	            return cdmRepository.getReferenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, Reference.class));
560
	        }else if (cdmBase.isInstanceOf(Classification.class)){
561
	            return cdmRepository.getClassificationService().saveOrUpdate(CdmBase.deproxy(cdmBase, Classification.class));
562
	        }else if (cdmBase.isInstanceOf(AgentBase.class)){
563
	            return cdmRepository.getAgentService().saveOrUpdate(CdmBase.deproxy(cdmBase, AgentBase.class));
564
	        }else if (cdmBase.isInstanceOf(Collection.class)){
565
	            return cdmRepository.getCollectionService().saveOrUpdate(CdmBase.deproxy(cdmBase, Collection.class));
566
	        }else if (cdmBase.isInstanceOf(DescriptionBase.class)){
567
	            return cdmRepository.getDescriptionService().saveOrUpdate(CdmBase.deproxy(cdmBase, DescriptionBase.class));
568
	        }else if (cdmBase.isInstanceOf(TaxonBase.class)){
569
	            return cdmRepository.getTaxonService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonBase.class));
570
	        }else if (cdmBase.isInstanceOf(TaxonName.class)){
571
	            return cdmRepository.getNameService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonName.class));
572
	        }else if (cdmBase.isInstanceOf(TaxonNode.class)){
573
                return cdmRepository.getTaxonNodeService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonNode.class));
574
            }else{
575
	            throw new IllegalArgumentException("Class not supported in save method: " + CdmBase.deproxy(cdmBase, CdmBase.class).getClass().getSimpleName());
576
	        }
577

    
578
	    }
579

    
580

    
581
	    protected SpecimenOrObservationBase findExistingSpecimen(String unitId, SpecimenImportStateBase state){
582
	        ICdmRepository cdmAppController = state.getConfig().getCdmAppController();
583
	        if(cdmAppController==null){
584
	            cdmAppController = this;
585
	        }
586
	        FindOccurrencesConfigurator config = new FindOccurrencesConfigurator();
587
	        config.setSignificantIdentifier(unitId);
588
	        Pager<SpecimenOrObservationBase> existingSpecimens = cdmAppController.getOccurrenceService().findByTitle(config);
589
	        if(!existingSpecimens.getRecords().isEmpty()){
590
	            if(existingSpecimens.getRecords().size()==1){
591
	                return existingSpecimens.getRecords().iterator().next();
592
	            }
593
	        }
594
	        return null;
595
	    }
596

    
597
	    protected abstract void importAssociatedUnits(STATE state, Object item, DerivedUnitFacade derivedUnitFacade);
598

    
599
	    /**
600
	     * getFacade : get the DerivedUnitFacade based on the recordBasis
601
	     * @param state
602
	     *
603
	     * @return DerivedUnitFacade
604
	     */
605
	    protected DerivedUnitFacade getFacade(STATE state) {
606
	        if (logger.isDebugEnabled()){
607
	            logger.info("getFacade()");
608
	        }
609
	        SpecimenOrObservationType type = null;
610

    
611
	        // create specimen
612
	        if (NB((state.getDataHolder().getRecordBasis())) != null) {
613
	            if (state.getDataHolder().getRecordBasis().toLowerCase().startsWith("s") || state.getDataHolder().getRecordBasis().toLowerCase().indexOf("specimen")>-1) {// specimen
614
	                type = SpecimenOrObservationType.PreservedSpecimen;
615
	            }
616
	            if (state.getDataHolder().getRecordBasis().toLowerCase().startsWith("o") ||state.getDataHolder().getRecordBasis().toLowerCase().indexOf("observation")>-1 ) {
617
	                type = SpecimenOrObservationType.Observation;
618
	            }
619
	            if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("fossil")>-1){
620
	                type = SpecimenOrObservationType.Fossil;
621
	            }
622
	            if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("living")>-1) {
623
	                type = SpecimenOrObservationType.LivingSpecimen;
624
	            }
625
	            if (type == null) {
626
	                logger.info("The basis of record does not seem to be known: " + state.getDataHolder().getRecordBasis());
627
	                type = SpecimenOrObservationType.DerivedUnit;
628
	            }
629
	            // TODO fossils?
630
	        } else {
631
	            logger.info("The basis of record is null");
632
	            type = SpecimenOrObservationType.DerivedUnit;
633
	        }
634
	        DerivedUnitFacade derivedUnitFacade = DerivedUnitFacade.NewInstance(type);
635
	        return derivedUnitFacade;
636
	    }
637

    
638
	    /**
639
	     * Look if the Institution does already exist
640
	     * @param institutionCode: a string with the institutioncode
641
	     * @param config : the configurator
642
	     * @return the Institution (existing or new)
643
	     */
644
	    protected Institution getInstitution(String institutionCode, STATE state) {
645
	        SpecimenImportConfiguratorBase config = state.getConfig();
646
	        Institution institution=null;
647
	        institution = (Institution)state.institutions.get(institutionCode);
648
	        if (institution != null){
649
	            return institution;
650
	        }
651
	        List<Institution> institutions;
652
	        try {
653
	            institutions = getAgentService().searchInstitutionByCode(institutionCode);
654

    
655
	        } catch (Exception e) {
656
	            institutions = new ArrayList<Institution>();
657
	            logger.warn(e);
658
	        }
659
	        if (institutions.size() > 0 && config.isReuseExistingMetaData()) {
660
	            for (Institution institut:institutions){
661
	                try{
662
	                    if (institut.getCode().equalsIgnoreCase(institutionCode)) {
663
	                        institution=institut;
664
	                        break;
665
	                    }
666
	                }catch(Exception e){logger.warn("no institution code in the db");}
667
	            }
668
	        }
669
	        if (logger.isDebugEnabled()){
670
	            if(institution !=null) {
671
	                logger.info("getinstitution " + institution.toString());
672
	            }
673
	        }
674
	        if (institution == null){
675
	            // create institution
676
	            institution = Institution.NewInstance();
677
	            institution.setCode(institutionCode);
678
	            institution.setTitleCache(institutionCode, true);
679
	            UUID uuid = save(institution, state);
680
	        }
681

    
682

    
683
	        state.institutions.put(institutionCode, institution);
684
	        return institution;
685
	    }
686

    
687
	    /**
688
	     * Look if the Collection does already exist
689
	     * @param collectionCode
690
	     * @param collectionCode: a string
691
	     * @param config : the configurator
692
	     * @return the Collection (existing or new)
693
	     */
694
	    protected Collection getCollection(Institution institution, String collectionCode, STATE state) {
695
	        SpecimenImportConfiguratorBase config = state.getConfig();
696
	        Collection collection = null;
697
	        List<Collection> collections;
698
	        collection = (Collection) state.collections.get(collectionCode);
699
	        if (collection != null){
700
	            return collection;
701
	        }
702
	        try {
703
	            collections = getCollectionService().searchByCode(collectionCode);
704
	        } catch (Exception e) {
705
	            collections = new ArrayList<Collection>();
706
	        }
707
	        if (collections.size() > 0 && config.isReuseExistingMetaData()) {
708
	            for (Collection coll:collections){
709
	                if (coll.getCode() != null && coll.getInstitute() != null
710
	                        && coll.getCode().equalsIgnoreCase(collectionCode) && coll.getInstitute().equals(institution)) {
711
	                    collection = coll;
712
	                    break;
713
	                }
714
	            }
715
	        }
716

    
717
	        if(collection == null){
718
	            collection =Collection.NewInstance();
719
	            collection.setCode(collectionCode);
720
	            collection.setInstitute(institution);
721
	            collection.setTitleCache(collectionCode);
722
	            UUID uuid = save(collection, state);
723
	        }
724

    
725

    
726

    
727
	        state.collections.put(collectionCode, collection);
728

    
729
	        return collection;
730
	    }
731

    
732
	    /**
733
	     * @param reference
734
	     * @param citationDetail
735
	     * @return
736
	     */
737
	    //FIXME this method is highly critical, because
738
	    //  * it will have serious performance and memory problems with large databases
739
	    //        (databases may easily have >1 Mio source records)
740
	    //  * it does not make sense to search for existing sources and then clone them
741
	    //    we need to search for existing references instead and use them (if exist)
742
	    //    for our new source.
743
	    protected IdentifiableSource getIdentifiableSource(Reference reference, String citationDetail) {
744

    
745
	      /*  List<IdentifiableSource> issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
746

    
747

    
748
	        if (reference != null){
749
	            try {
750
	                for (OriginalSourceBase<?> osb: issTmp){
751
	                    if (osb.getCitation() != null && osb.getCitation().getTitleCache().equalsIgnoreCase(reference.getTitleCache())){
752
	                        String osbDetail = osb.getCitationMicroReference();
753
	                        if ((StringUtils.isBlank(osbDetail) && StringUtils.isBlank(citationDetail))
754
	                                || (osbDetail != null && osbDetail.equalsIgnoreCase(citationDetail)) ) {
755
//	                            System.out.println("REFERENCE FOUND RETURN EXISTING SOURCE");
756
	                            return (IdentifiableSource) osb.clone();
757
	                        }
758
	                    }
759
	                }
760
	            } catch (CloneNotSupportedException e) {
761
	                throw new RuntimeException(e);
762
	            } catch (Exception e1){
763
	                e1.printStackTrace();
764
	            }
765
	        }
766
	*/
767
	        IdentifiableSource sour = IdentifiableSource.NewInstance(OriginalSourceType.Import,null,null, reference,citationDetail);
768
	        return sour;
769
	    }
770

    
771
	    /**
772
	     * Add the hierarchy for a Taxon(add higher taxa)
773
	     * @param classification
774
	     * @param taxon: a taxon to add as a node
775
	     * @param state: the ABCD import state
776
	     */
777
	    protected void addParentTaxon(Taxon taxon, STATE state, boolean preferredFlag, Classification classification){
778
	        INonViralName  nvname = taxon.getName();
779
	        Rank rank = nvname.getRank();
780
	        Taxon genus =null;
781
	        Taxon subgenus =null;
782
	        Taxon species = null;
783
	        Taxon subspecies = null;
784
	        Taxon parent = null;
785
	        if(rank!=null){
786
	            if (rank.isLower(Rank.GENUS() )){
787
	                String genusOrUninomial = nvname.getGenusOrUninomial();
788
	                TaxonName taxonName = getOrCreateTaxonName(genusOrUninomial, Rank.GENUS(), preferredFlag, state, -1);
789
	                genus = getOrCreateTaxonForName(taxonName, state);
790
	                if (genus == null){
791
	                    logger.debug("The genus should not be null " + taxonName);
792
	                }
793
	                if (preferredFlag) {
794
	                    parent = linkParentChildNode(null, genus, classification, state);
795
	                }
796

    
797
	            }
798
	            if (rank.isLower(Rank.SUBGENUS())){
799
	                String prefix = nvname.getGenusOrUninomial();
800
	                String name = nvname.getInfraGenericEpithet();
801
	                if (name != null){
802
	                    TaxonName taxonName = getOrCreateTaxonName(prefix+" "+name, Rank.SUBGENUS(), preferredFlag, state, -1);
803
	                    subgenus = getOrCreateTaxonForName(taxonName, state);
804
	                    if (preferredFlag) {
805
	                        parent = linkParentChildNode(genus, subgenus, classification, state);
806
	                    }            }
807
	            }
808
	            if (rank.isLower(Rank.SPECIES())){
809
	                if (subgenus!=null){
810
	                    String prefix = nvname.getGenusOrUninomial();
811
	                    String name = nvname.getInfraGenericEpithet();
812
	                    String spe = nvname.getSpecificEpithet();
813
	                    if (spe != null){
814
	                        TaxonName taxonName = getOrCreateTaxonName(prefix+" "+name+" "+spe, Rank.SPECIES(), preferredFlag, state, -1);
815
	                        species = getOrCreateTaxonForName(taxonName, state);
816
	                        if (preferredFlag) {
817
	                            parent = linkParentChildNode(subgenus, species, classification, state);
818
	                        }
819
	                    }
820
	                }
821
	                else{
822
	                    String prefix = nvname.getGenusOrUninomial();
823
	                    String name = nvname.getSpecificEpithet();
824
	                    if (name != null){
825
	                        TaxonName taxonName = getOrCreateTaxonName(prefix+" "+name, Rank.SPECIES(), preferredFlag, state, -1);
826
	                        species = getOrCreateTaxonForName(taxonName, state);
827
	                        if (preferredFlag) {
828
	                            parent = linkParentChildNode(genus, species, classification, state);
829
	                        }
830
	                    }
831
	                }
832
	            }
833
	            if (rank.isLower(Rank.INFRASPECIES())){
834
	                TaxonName taxonName = getOrCreateTaxonName(nvname.getFullTitleCache(), Rank.SUBSPECIES(), preferredFlag, state, -1);
835
	                subspecies = getOrCreateTaxonForName(taxonName, state);
836
	                if (preferredFlag) {
837
	                    parent = linkParentChildNode(species, subspecies, classification, state);
838
	                }
839
	            }
840
	        }
841
	        if (preferredFlag && parent!=taxon ) {
842
	            linkParentChildNode(parent, taxon, classification, state);
843
	        }
844
	    }
845

    
846
	    /**
847
	     * Link a parent to a child and save it in the current classification
848
	     * @param parent: the higher Taxon
849
	     * @param child : the lower (or current) Taxon
850
	     * return the Taxon from the new created Node
851
	     * @param classification
852
	     * @param state
853
	     */
854
	    protected Taxon linkParentChildNode(Taxon parent, Taxon child, Classification classification, STATE state) {
855
	        TaxonNode node =null;
856
	        if (parent != null) {
857
	            parent = (Taxon) getTaxonService().find(parent.getUuid());
858
	            child = (Taxon) getTaxonService().find(child.getUuid());
859
	            //here we do not have to check if the taxon nodes already exists
860
	            //this is done by classification.addParentChild()
861
	            //do not add child node if it already exists
862
	            if(hasTaxonNodeInClassification(child, classification)){
863
	                return child;
864
	            }
865
	            else{
866
	                node = classification.addParentChild(parent, child, state.getRef(), "");
867
	                save(node, state);
868
	            }
869
	        }
870
	        else {
871
	            if (child == null){
872
	                logger.debug("The child should not be null!");
873
	            }
874
	            child = (Taxon) getTaxonService().find(child.getUuid());
875
	            //do not add child node if it already exists
876
	            if(hasTaxonNodeInClassification(child, classification)){
877
	                return child;
878
	            }
879
	            else{
880
	                node = classification.addChildTaxon(child, state.getRef(), null);
881
	                save(node, state);
882
	            }
883
	        }
884
	        if(node!=null){
885
	            state.getReport().addTaxonNode(node);
886
	            return node.getTaxon();
887
	        }
888
	        String message = "Could not create taxon node for " +child;
889
	        state.getReport().addInfoMessage(message);
890
	        logger.warn(message);
891
	        return null;
892
	    }
893

    
894
	    protected Taxon getOrCreateTaxonForName(TaxonName taxonName, STATE state){
895
	        if (taxonName != null){
896
    	        Set<Taxon> acceptedTaxa = taxonName.getTaxa();
897
    	        if(acceptedTaxa.size()>0){
898
    	            Taxon firstAcceptedTaxon = acceptedTaxa.iterator().next();
899
    	            if(acceptedTaxa.size()>1){
900
    	                String message = "More than one accepted taxon was found for taxon name: "
901
    	                        + taxonName.getTitleCache() + "!\n" + firstAcceptedTaxon + "was chosen for "+state.getDerivedUnitBase();
902
    	                state.getReport().addInfoMessage(message);
903
    	                logger.warn(message);
904
    	            }
905
    	            else{
906
    	                return firstAcceptedTaxon;
907
    	            }
908
    	        }
909
    	        else{
910
    	            Set<TaxonBase> taxonAndSynonyms = taxonName.getTaxonBases();
911
    	            for (TaxonBase taxonBase : taxonAndSynonyms) {
912
    	                if(taxonBase.isInstanceOf(Synonym.class)){
913
    	                    Synonym synonym = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
914
    	                    Taxon acceptedTaxonOfSynonym = synonym.getAcceptedTaxon();
915
    	                    if(acceptedTaxonOfSynonym == null){
916
    	                        String message = "No accepted taxon could be found for taxon name: "
917
    	                                + taxonName.getTitleCache()
918
    	                                + "!";
919
    	                        state.getReport().addInfoMessage(message);
920
    	                        logger.warn(message);
921
    	                    }
922
    	                    else{
923
    	                        return acceptedTaxonOfSynonym;
924
    	                    }
925
    	                }
926
    	            }
927
    	        }
928
    	        Taxon taxon = Taxon.NewInstance(taxonName, state.getRef());
929
    	        save(taxon, state);
930
    	        state.getReport().addTaxon(taxon);
931
    	        logger.info("Created new taxon "+ taxon);
932
    	        return taxon;
933
	        }
934
	        return null;
935

    
936
	    }
937

    
938
	    private boolean hasTaxonNodeInClassification(Taxon taxon, Classification classification){
939
	        if(taxon.getTaxonNodes()!=null){
940
	            for (TaxonNode node : taxon.getTaxonNodes()){
941
	                if(node.getClassification().equals(classification)){
942
	                    return true;
943
	                }
944
	            }
945
	        }
946
	        return false;
947
	    }
948

    
949
	    /**
950
	     * HandleIdentifications : get the scientific names present in the ABCD
951
	     * document and store link them with the observation/specimen data
952
	     * @param state: the current ABCD import state
953
	     * @param derivedUnitFacade : the current derivedunitfacade
954
	     */
955
	    protected void handleIdentifications(STATE state, DerivedUnitFacade derivedUnitFacade) {
956
	       SpecimenImportConfiguratorBase config = state.getConfig();
957

    
958

    
959
	        String scientificName = "";
960
	        boolean preferredFlag = false;
961

    
962
	        if (state.getDataHolder().getNomenclatureCode() == ""){
963
	            if (config.getNomenclaturalCode() != null){
964
	                if (config.getNomenclaturalCode() != null){
965
	                    state.getDataHolder().setNomenclatureCode(config.getNomenclaturalCode().toString());
966

    
967
	                }
968
	            }
969
	        }
970

    
971
	        for (int i = 0; i < state.getDataHolder().getIdentificationList().size(); i++) {
972
	            Identification identification = state.getDataHolder().getIdentificationList().get(i);
973
	            scientificName = identification.getScientificName().replaceAll(" et ", " & ");
974

    
975
	            String preferred = identification.getPreferred();
976
	            preferredFlag = false;
977
	            if (preferred != null){
978
    	            if (preferred.equals("1") || preferred.toLowerCase().indexOf("true") != -1 || state.getDataHolder().getIdentificationList().size()==1) {
979
    	                preferredFlag = true;
980
    	            }
981

    
982
	            }
983
	            if (identification.getCode() != null){
984
    	            if (identification.getCode().indexOf(':') != -1) {
985
    	                state.getDataHolder().setNomenclatureCode(identification.getCode().split(COLON)[1]);
986
    	            }
987
    	            else{
988
    	                state.getDataHolder().setNomenclatureCode(identification.getCode());
989
    	            }
990
	            }
991
	            TaxonName taxonName = getOrCreateTaxonName(scientificName, null, preferredFlag, state, i);
992
	            Taxon taxon = getOrCreateTaxonForName(taxonName, state);
993
	            addTaxonNode(taxon, state,preferredFlag);
994
	            linkDeterminationEvent(state, taxon, preferredFlag, derivedUnitFacade, identification.getIdentifier(), identification.getDate());
995
	        }
996
	    }
997

    
998
	    /**
999
	     * @param taxon : a taxon to add as a node
1000
	     * @param state : the ABCD import state
1001
	     */
1002
	    protected void addTaxonNode(Taxon taxon, STATE state, boolean preferredFlag) {
1003
	        SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
1004
	        logger.info("link taxon to a taxonNode "+taxon.getTitleCache());
1005
	        //only add nodes if not already existing in current classification or default classification
1006

    
1007
	        //check if node exists in current classification
1008
	        //NOTE: we cannot use hasTaxonNodeInClassification() here because we are first creating it here
1009
	        if (!existsInClassification(taxon,state.getClassification(), state)){
1010
	            if(config.isMoveNewTaxaToDefaultClassification()){
1011
	                //check if node exists in default classification
1012
	                if (!existsInClassification(taxon, state.getDefaultClassification(), state)){
1013
	                    addParentTaxon(taxon, state, preferredFlag, state.getDefaultClassification());
1014
	                }
1015
	            }
1016
	            else {
1017
	                //add non-existing taxon to current classification
1018
	                addParentTaxon(taxon, state, preferredFlag, state.getClassification());
1019
	            }
1020
	        }
1021
	    }
1022

    
1023

    
1024
	    private boolean existsInClassification(Taxon taxon, Classification classification, STATE state){
1025
	        boolean exist = false;
1026
	        ICdmRepository cdmAppController = state.getConfig().getCdmAppController();
1027
            if(cdmAppController==null){
1028
                cdmAppController = this;
1029
            }
1030
            if (classification != null){
1031
                if (!taxon.getTaxonNodes().isEmpty()){
1032
                    for (TaxonNode node:taxon.getTaxonNodes()){
1033
                        if (node.getClassification().equals(classification)){
1034
                            return true;
1035
                        }
1036
                    }
1037
                }
1038
// we do not need this because we already searched for taxa in db in the previous steps
1039
//    	        List<UuidAndTitleCache<TaxonNode>> uuidAndTitleCacheOfAllTaxa = cdmAppController.getClassificationService().getTaxonNodeUuidAndTitleCacheOfAcceptedTaxaByClassification(classification.getUuid());
1040
//    	        if (uuidAndTitleCacheOfAllTaxa != null){
1041
//        	        for (UuidAndTitleCache p : uuidAndTitleCacheOfAllTaxa){
1042
//        	            try{
1043
//        	                if(p.getTitleCache().equals(taxon.getTitleCache())) {
1044
//        	                    exist = true;
1045
//        	                }
1046
//        	            }
1047
//        	            catch(Exception e){
1048
//        	                logger.warn("TaxonNode doesn't seem to have a taxon");
1049
//        	            }
1050
//        	        }
1051
//    	        }
1052
            }
1053
	        return exist;
1054
	    }
1055

    
1056
	    /**
1057
	     * join DeterminationEvent to the Taxon Object
1058
	     * @param state : the ABCD import state
1059
	     * @param taxon: the current Taxon
1060
	     * @param preferredFlag :if the current name is preferred
1061
	     * @param derivedFacade : the derived Unit Facade
1062
	     */
1063
	    @SuppressWarnings("rawtypes")
1064
        protected void linkDeterminationEvent(STATE state, Taxon taxon, boolean preferredFlag,  DerivedUnitFacade derivedFacade, String identifierStr, String dateStr) {
1065
	        SpecimenImportConfiguratorBase config = state.getConfig();
1066
	        if (logger.isDebugEnabled()){
1067
	            logger.info("start linkdetermination with taxon:" + taxon.getUuid()+", "+taxon);
1068
	        }
1069

    
1070
	        DeterminationEvent determinationEvent = DeterminationEvent.NewInstance();
1071
	        determinationEvent.setTaxonName(taxon.getName());
1072
	        determinationEvent.setPreferredFlag(preferredFlag);
1073

    
1074

    
1075
	        determinationEvent.setIdentifiedUnit(state.getDerivedUnitBase());
1076
	        if (state.getPersonStore().get(identifierStr) != null){
1077
	            determinationEvent.setActor((AgentBase)state.getPersonStore().get(identifierStr));
1078
	        } else if (identifierStr != null){
1079
	            Person identifier = Person.NewTitledInstance(identifierStr);
1080
	            determinationEvent.setActor(identifier);
1081
	        }
1082
	        if (dateStr != null){
1083
	            determinationEvent.setTimeperiod(TimePeriodParser.parseString(dateStr));
1084
	        }
1085
	        state.getDerivedUnitBase().addDetermination(determinationEvent);
1086

    
1087
	        if (logger.isDebugEnabled()){
1088
	            logger.debug("NB TYPES INFO: "+ state.getDataHolder().getStatusList().size());
1089
	        }
1090
	        for (SpecimenTypeDesignationStatus specimenTypeDesignationstatus : state.getDataHolder().getStatusList()) {
1091
	            if (specimenTypeDesignationstatus != null) {
1092
	                if (logger.isDebugEnabled()){
1093
	                    logger.debug("specimenTypeDesignationstatus :"+ specimenTypeDesignationstatus);
1094
	                }
1095

    
1096
	                ICdmRepository cdmAppController = config.getCdmAppController();
1097
	                if(cdmAppController == null){
1098
	                    cdmAppController = this;
1099
	                }
1100
	                specimenTypeDesignationstatus = (SpecimenTypeDesignationStatus) cdmAppController.getTermService().find(specimenTypeDesignationstatus.getUuid());
1101
	                //Designation
1102
	                TaxonName name = taxon.getName();
1103
	                SpecimenTypeDesignation designation = SpecimenTypeDesignation.NewInstance();
1104

    
1105
	                designation.setTypeStatus(specimenTypeDesignationstatus);
1106
	                designation.setTypeSpecimen(state.getDerivedUnitBase());
1107
	                name.addTypeDesignation(designation, true);
1108
	            }
1109
	        }
1110

    
1111
	        for (String[] fullReference : state.getDataHolder().getReferenceList()) {
1112

    
1113

    
1114
	            String strReference=fullReference[0];
1115
	            String citationDetail = fullReference[1];
1116
	            String citationURL = fullReference[2];
1117
	            List<Reference> references = getReferenceService().listByTitle(Reference.class, "strReference", MatchMode.EXACT, null, null, null, null, null);
1118

    
1119
	            if (!references.isEmpty()){
1120
	                Reference reference = null;
1121
	                for (Reference refe: references) {
1122
	                    if (refe.getTitleCache().equalsIgnoreCase(strReference)) {
1123
	                        reference =refe;
1124
	                        break;
1125
	                    }
1126
	                }
1127
	                if (reference ==null){
1128
	                    reference = ReferenceFactory.newGeneric();
1129
	                    reference.setTitleCache(strReference, true);
1130
	                    save(reference, state);
1131
	                }
1132
	                determinationEvent.addReference(reference);
1133
	            }
1134
	        }
1135
	        save(state.getDerivedUnitBase(), state);
1136

    
1137
	        if (config.isAddIndividualsAssociationsSuchAsSpecimenAndObservations() && preferredFlag) {
1138
	            //do not add IndividualsAssociation to non-preferred taxa
1139
	            if (logger.isDebugEnabled()){
1140
	                logger.debug("isDoCreateIndividualsAssociations");
1141
	            }
1142

    
1143
	            makeIndividualsAssociation(state, taxon, determinationEvent);
1144

    
1145
	            save(state.getDerivedUnitBase(), state);
1146
	        }
1147
	    }
1148

    
1149
	    /**
1150
	     * create and link each association (specimen, observation..) to the accepted taxon
1151
	     * @param state : the ABCD import state
1152
	     * @param taxon: the current Taxon
1153
	     * @param determinationEvent:the determinationevent
1154
	     */
1155
	    protected void makeIndividualsAssociation(STATE state, Taxon taxon, DeterminationEvent determinationEvent) {
1156
	        SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
1157
	        SpecimenUserInteraction sui = config.getSpecimenUserInteraction();
1158

    
1159
	        if (logger.isDebugEnabled()){
1160
	            logger.info("MAKE INDIVIDUALS ASSOCIATION");
1161
	        }
1162

    
1163
	        TaxonDescription taxonDescription = null;
1164
	        Set<TaxonDescription> descriptions= taxon.getDescriptions();
1165
	       if (!descriptions.isEmpty()){ taxonDescription = descriptions.iterator().next();}
1166

    
1167
//	            for (TaxonDescription description : descriptions){
1168
//	                Set<IdentifiableSource> sources =  new HashSet<>();
1169
//	                sources.addAll(description.getTaxon().getSources());
1170
//	                sources.addAll(description.getSources());
1171
//	                for (IdentifiableSource source:sources){
1172
//	                    if(state.getRef().equals(source.getCitation())) {
1173
//	                        taxonDescription = description;
1174
//	                    }
1175
//	                }
1176
//	            }
1177
	      //  }
1178
	        if (taxonDescription == null){
1179
	            taxonDescription = TaxonDescription.NewInstance(taxon, false);
1180
	            if(sourceNotLinkedToElement(taxonDescription,state.getRef(),null)) {
1181
	                taxonDescription.addSource(OriginalSourceType.Import, null, null, state.getRef(), null);
1182
	            }
1183
	            state.setDescriptionGroup(taxonDescription);
1184
	            taxon.addDescription(taxonDescription);
1185
	        }
1186

    
1187
	        //PREPARE REFERENCE QUESTIONS
1188

    
1189
	        Map<String,OriginalSourceBase<?>> sourceMap = new HashMap<String, OriginalSourceBase<?>>();
1190

    
1191
	        List<IdentifiableSource> issTmp = new ArrayList<>();//getCommonService().list(IdentifiableSource.class, null, null, null, null);
1192
	        List<DescriptionElementSource> issTmp2 = new ArrayList<>();//getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1193

    
1194
	        Set<OriginalSourceBase> osbSet = new HashSet<OriginalSourceBase>();
1195
	        if(issTmp2!=null) {
1196
	            osbSet.addAll(issTmp2);
1197
	        }
1198
	        if(issTmp!=null) {
1199
	            osbSet.addAll(issTmp);
1200
	        }
1201

    
1202

    
1203
	        addToSourceMap(sourceMap, osbSet);
1204

    
1205
//	        if (((Abcd206ImportConfigurator) state.getConfig()).isInteractWithUser()){
1206
//	            List<OriginalSourceBase<?>> res = null;
1207
//	            if(!state.isDescriptionSourcesSet()){
1208
//	                res = sui.askForSource(sourceMap, "the description group ("+taxon+")",
1209
//	                        "The current reference is "+state.getRef().getTitleCache(),getReferenceService(), state.getDataHolder().docSources);
1210
//	                state.setDescriptionRefs(res);
1211
//	                state.setDescriptionSourcesSet(true);
1212
//	            }
1213
//	            else{
1214
//	                res=state.getDescriptionRefs();
1215
//	            }
1216
//	            if(res !=null) {
1217
//	                for (OriginalSourceBase<?> sour:res){
1218
//	                    if(sour.isInstanceOf(IdentifiableSource.class)){
1219
//	                        try {
1220
//	                            if(sourceNotLinkedToElement(taxonDescription,sour)) {
1221
//	                                taxonDescription.addSource((IdentifiableSource)sour.clone());
1222
//	                            }
1223
//	                        } catch (CloneNotSupportedException e) {
1224
//	                            logger.warn("no cloning?");
1225
//	                        }
1226
//	                    }else{
1227
//	                        if(sourceNotLinkedToElement(taxonDescription,sour)) {
1228
//	                            taxonDescription.addSource(OriginalSourceType.Import,null, null, sour.getCitation(),sour.getCitationMicroReference());
1229
//	                        }
1230
//	                    }
1231
//	                }
1232
//	            }
1233
//	        }
1234
//	        else {
1235
	            if(sourceNotLinkedToElement(taxonDescription,state.getRef(),null)) {
1236
	                taxonDescription.addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1237
	            }
1238
//	        }
1239
	        state.setDescriptionGroup(taxonDescription);
1240

    
1241
	        IndividualsAssociation indAssociation = IndividualsAssociation.NewInstance();
1242
	        Feature feature = makeFeature(state.getDerivedUnitBase());
1243
	        indAssociation.setAssociatedSpecimenOrObservation(state.getDerivedUnitBase());
1244
	        indAssociation.setFeature(feature);
1245
//
1246
//	        if (((Abcd206ImportConfigurator) state.getConfig()).isInteractWithUser()){
1247
//	            sourceMap = new HashMap<String, OriginalSourceBase<?>>();
1248
//
1249
//	            issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
1250
//	            issTmp2 = getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1251
//
1252
//	            osbSet = new HashSet<OriginalSourceBase>();
1253
//	            if(issTmp2!=null) {
1254
//	                osbSet.addAll(issTmp2);
1255
//	            }
1256
//	            if(issTmp!=null) {
1257
//	                osbSet.addAll(issTmp);
1258
//	            }
1259
//
1260
//
1261
//	            addToSourceMap(sourceMap, osbSet);
1262
//
1263
//	            List<OriginalSourceBase<?>> sources =null;
1264
//	            if(!state.isAssociationSourcesSet()) {
1265
//	                sources = sui.askForSource(sourceMap,  "descriptive element (association) ",taxon.toString(),
1266
//	                        getReferenceService(),state.getDataHolder().getDocSources());
1267
//	                state.setAssociationRefs(sources);
1268
//	                state.setAssociationSourcesSet(true);
1269
//	            }
1270
//	            else{
1271
//	                sources=state.getAssociationRefs();
1272
//	            }
1273
//	            if(sources !=null) {
1274
//	                for (OriginalSourceBase<?> source: sources) {
1275
//	                    if(source !=null) {
1276
//	                        if(source.isInstanceOf(DescriptionElementSource.class)){
1277
//	                            try {
1278
//	                                if(sourceNotLinkedToElement(indAssociation,source)) {
1279
//	                                    indAssociation.addSource((DescriptionElementSource)source.clone());
1280
//	                                }
1281
//	                            } catch (CloneNotSupportedException e) {
1282
//	                                logger.warn("clone forbidden?");
1283
//	                            }
1284
//	                        }else{
1285
//	                            if(sourceNotLinkedToElement(indAssociation,source)) {
1286
//	                                indAssociation.addSource(OriginalSourceType.Import,null, null, source.getCitation(),source.getCitationMicroReference());
1287
//	                            }
1288
//	                            try {
1289
//	                                if(sourceNotLinkedToElement(state.getDerivedUnitBase(), source)) {
1290
//	                                    state.getDerivedUnitBase().addSource((IdentifiableSource) source.clone());
1291
//	                                }
1292
//	                            } catch (CloneNotSupportedException e) {
1293
//	                                // TODO Auto-generated catch block
1294
//	                                e.printStackTrace();
1295
//	                            }
1296
//	                        }
1297
//
1298
//	                    }
1299
//	                }
1300
//	            }
1301
//	        }else {
1302
	            if(sourceNotLinkedToElement(indAssociation,state.getRef(),null)) {
1303
	                indAssociation.addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1304
	            }
1305
	            if(sourceNotLinkedToElement(state.getDerivedUnitBase(), state.getRef(),null)) {
1306
	                state.getDerivedUnitBase().addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1307
	            }
1308
	            for (Reference citation : determinationEvent.getReferences()) {
1309
	                if(sourceNotLinkedToElement(indAssociation,citation,null))
1310
	                {
1311
	                    indAssociation.addSource(DescriptionElementSource.NewInstance(OriginalSourceType.Import, null, null, citation, null));
1312
	                }
1313
	                if(sourceNotLinkedToElement(state.getDerivedUnitBase(), state.getRef(),null)) {
1314
	                    state.getDerivedUnitBase().addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1315
	                }
1316
	            }
1317
	   //     }
1318

    
1319
	        taxonDescription.addElement(indAssociation);
1320

    
1321
	        save(taxonDescription, state);
1322
	        save(taxon, state);
1323
	        state.getReport().addDerivate(state.getDerivedUnitBase(), config);
1324
	        state.getReport().addIndividualAssociation(taxon, state.getDataHolder().getUnitID(), state.getDerivedUnitBase());
1325
	    }
1326

    
1327
	    /**
1328
	     * @param derivedUnitBase2
1329
	     * @param ref2
1330
	     * @param object
1331
	     * @return
1332
	     */
1333
	    private boolean sourceNotLinkedToElement(DerivedUnit derivedUnitBase2, Reference b, String d) {
1334
	        Set<IdentifiableSource> linkedSources = derivedUnitBase2.getSources();
1335
	        for (IdentifiableSource is:linkedSources){
1336
	            Reference a = is.getCitation();
1337
	            String c = is.getCitationMicroReference();
1338

    
1339
	            boolean refMatch=false;
1340
	            boolean microMatch=false;
1341

    
1342
	            try{
1343
	                if (a==null && b==null) {
1344
	                    refMatch=true;
1345
	                }
1346
	                if (a!=null && b!=null) {
1347
	                    if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1348
	                        refMatch=true;
1349
	                    }
1350
	                }
1351
	            }catch(Exception e){}
1352

    
1353

    
1354
	            try{
1355
	                if (c==null && d==null) {
1356
	                    microMatch=true;
1357
	                }
1358
	                if(c!=null && d!=null) {
1359
	                    if(c.equalsIgnoreCase(d)) {
1360
	                        microMatch=true;
1361
	                    }
1362
	                }
1363
	            }
1364
	            catch(Exception e){}
1365

    
1366
	            if (microMatch && refMatch) {
1367
	                return false;
1368
	            }
1369

    
1370

    
1371
	        }
1372
	        return true;
1373
	    }
1374

    
1375
	    private <T extends OriginalSourceBase<?>> boolean  sourceNotLinkedToElement(ISourceable<T> sourcable, Reference reference, String microReference) {
1376
	        Set<T> linkedSources = sourcable.getSources();
1377
	        for (T is:linkedSources){
1378
	            Reference unitReference = is.getCitation();
1379
	            String unitMicroReference = is.getCitationMicroReference();
1380

    
1381
	            boolean refMatch=false;
1382
	            boolean microMatch=false;
1383

    
1384
	            try{
1385
	                if (unitReference==null && reference==null) {
1386
	                    refMatch=true;
1387
	                }
1388
	                if (unitReference!=null && reference!=null) {
1389
	                    if (unitReference.getTitleCache().equalsIgnoreCase(reference.getTitleCache())) {
1390
	                        refMatch=true;
1391
	                    }
1392
	                }
1393
	            }catch(Exception e){}
1394

    
1395
	            try{
1396
	                if (unitMicroReference==null && microReference==null) {
1397
	                    microMatch=true;
1398
	                }
1399
	                if(unitMicroReference!=null && microReference!=null) {
1400
	                    if(unitMicroReference.equalsIgnoreCase(microReference)) {
1401
	                        microMatch=true;
1402
	                    }
1403
	                }
1404
	            }
1405
	            catch(Exception e){}
1406

    
1407
	            if (microMatch && refMatch) {
1408
	                return false;
1409
	            }
1410
	        }
1411
	        return true;
1412
	    }
1413

    
1414
	    /**
1415
	     * look for the Feature object (FieldObs, Specimen,...)
1416
	     * @param unit : a specimen or obersvation base
1417
	     * @return the corresponding Feature
1418
	     */
1419
	    private Feature makeFeature(SpecimenOrObservationBase<?> unit) {
1420
	        SpecimenOrObservationType type = unit.getRecordBasis();
1421

    
1422

    
1423

    
1424
	        if (type.isFeatureObservation()){
1425
	            return Feature.OBSERVATION();
1426
	        }else if (type.isFeatureSpecimen()){
1427
	            return Feature.SPECIMEN();
1428
	        }else if (type == SpecimenOrObservationType.DerivedUnit){
1429
	            return Feature.OBSERVATION();
1430
	            //            return getFeature("Specimen or observation");
1431
	        }else{
1432
	            String message = "Unhandled record basis '%s' for defining individuals association feature type. Use default.";
1433
	            logger.warn(String.format(message, type.getMessage()));
1434
	            return Feature.OBSERVATION();
1435
	            //            return getFeature("Specimen or observation");
1436

    
1437
	        }
1438
	    }
1439

    
1440

    
1441
	    /**
1442
	     * @param sourceMap
1443
	     * @param osbSet
1444
	     */
1445
	    protected void addToSourceMap(Map<String, OriginalSourceBase<?>> sourceMap, Set<OriginalSourceBase> osbSet) {
1446
	        for( OriginalSourceBase<?> osb:osbSet) {
1447
	            if(osb.getCitation()!=null && osb.getCitationMicroReference() !=null  && !osb.getCitationMicroReference().isEmpty()) {
1448
	                try{
1449
	                    sourceMap.put(osb.getCitation().getTitleCache()+ "---"+osb.getCitationMicroReference(),osb);
1450
	                }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1451
	            } else if(osb.getCitation()!=null){
1452
	                try{
1453
	                    sourceMap.put(osb.getCitation().getTitleCache(),osb);
1454
	                }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1455
	            }
1456
	        }
1457
	    }
1458

    
1459

    
1460
}
(2-2/7)