Project

General

Profile

Download (64.5 KB) Statistics
| Branch: | Tag: | Revision:
1
/**
2
* Copyright (C) 2007 EDIT
3
* European Distributed Institute of Taxonomy
4
* http://www.e-taxonomy.eu
5
*
6
* The contents of this file are subject to the Mozilla Public License Version 1.1
7
* See LICENSE.TXT at the top of this package for the full license terms.
8
*/
9

    
10
package eu.etaxonomy.cdm.io.specimen;
11

    
12

    
13
import java.util.ArrayList;
14
import java.util.HashMap;
15
import java.util.HashSet;
16
import java.util.List;
17
import java.util.Map;
18
import java.util.Set;
19
import java.util.UUID;
20

    
21
import org.apache.log4j.Logger;
22

    
23
import eu.etaxonomy.cdm.api.application.ICdmRepository;
24
import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
25
import eu.etaxonomy.cdm.api.service.config.FindOccurrencesConfigurator;
26
import eu.etaxonomy.cdm.api.service.pager.Pager;
27
import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
28
import eu.etaxonomy.cdm.io.common.CdmImportBase;
29
import eu.etaxonomy.cdm.io.common.IImportConfigurator;
30
import eu.etaxonomy.cdm.io.specimen.abcd206.in.Identification;
31
import eu.etaxonomy.cdm.io.specimen.abcd206.in.SpecimenImportReport;
32
import eu.etaxonomy.cdm.model.agent.AgentBase;
33
import eu.etaxonomy.cdm.model.agent.Institution;
34
import eu.etaxonomy.cdm.model.agent.Person;
35
import eu.etaxonomy.cdm.model.agent.Team;
36
import eu.etaxonomy.cdm.model.common.CdmBase;
37
import eu.etaxonomy.cdm.model.common.ISourceable;
38
import eu.etaxonomy.cdm.model.common.IdentifiableSource;
39
import eu.etaxonomy.cdm.model.common.LanguageString;
40
import eu.etaxonomy.cdm.model.common.OriginalSourceBase;
41
import eu.etaxonomy.cdm.model.common.OriginalSourceType;
42
import eu.etaxonomy.cdm.model.description.DescriptionBase;
43
import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
44
import eu.etaxonomy.cdm.model.description.Feature;
45
import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
46
import eu.etaxonomy.cdm.model.description.TaxonDescription;
47
import eu.etaxonomy.cdm.model.name.INonViralName;
48
import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
49
import eu.etaxonomy.cdm.model.name.Rank;
50
import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
51
import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
52
import eu.etaxonomy.cdm.model.name.TaxonName;
53
import eu.etaxonomy.cdm.model.name.TaxonNameFactory;
54
import eu.etaxonomy.cdm.model.occurrence.Collection;
55
import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
56
import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
57
import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
58
import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationType;
59
import eu.etaxonomy.cdm.model.reference.Reference;
60
import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
61
import eu.etaxonomy.cdm.model.taxon.Classification;
62
import eu.etaxonomy.cdm.model.taxon.Synonym;
63
import eu.etaxonomy.cdm.model.taxon.Taxon;
64
import eu.etaxonomy.cdm.model.taxon.TaxonBase;
65
import eu.etaxonomy.cdm.model.taxon.TaxonNode;
66
import eu.etaxonomy.cdm.persistence.query.MatchMode;
67
import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
68
import eu.etaxonomy.cdm.strategy.parser.ParserProblem;
69
import eu.etaxonomy.cdm.strategy.parser.TimePeriodParser;
70

    
71

    
72
/**
73
 * @author p.kelbert
74
 * @created 20.10.2008
75
 */
76
public abstract class SpecimenImportBase<CONFIG extends IImportConfigurator, STATE extends SpecimenImportStateBase>
77
        extends CdmImportBase<CONFIG, STATE> {
78

    
79
    private static final long serialVersionUID = 4423065367998125678L;
80
    private static final Logger logger = Logger.getLogger(SpecimenImportBase.class);
81

    
82
	protected static final UUID SPECIMEN_SCAN_TERM = UUID.fromString("acda15be-c0e2-4ea8-8783-b9b0c4ad7f03");
83

    
84
	private static final String COLON = ":";
85

    
86

    
87
	@Override
88
    protected abstract void doInvoke(STATE state);
89

    
90
	/**
91
	 * Handle a single unit
92
	 * @param state
93
	 * @param item
94
	 */
95
	protected abstract void handleSingleUnit(STATE state, Object item) ;
96

    
97

    
98

    
99
	protected TaxonName getOrCreateTaxonName(String scientificName, Rank rank, boolean preferredFlag, STATE state, int unitIndexInAbcdFile){
100
	    TaxonName taxonName = null;
101
        SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
102

    
103
        //check atomised name data for rank
104
        //new name will be created
105
        TaxonName atomisedTaxonName = null;
106
        if (rank==null && unitIndexInAbcdFile>=0 && ((state.getDataHolder().getAtomisedIdentificationList() != null && !state.getDataHolder().getAtomisedIdentificationList().isEmpty())|| state.getDataHolder().getAtomisedIdentificationList().size() > 0)) {
107
            atomisedTaxonName = setTaxonNameByType(state.getDataHolder().getAtomisedIdentificationList().get(unitIndexInAbcdFile), scientificName, state);
108
            if(atomisedTaxonName!=null){
109
                rank = atomisedTaxonName.getRank();
110
            }
111
        }
112
        if(config.isReuseExistingTaxaWhenPossible()){
113
            TaxonName parsedName = atomisedTaxonName;
114
            if(parsedName==null){
115

    
116
                parsedName = parseScientificName(scientificName, state, state.getReport(), rank);
117

    
118
            }
119
            atomisedTaxonName = parsedName;
120
            if(config.isIgnoreAuthorship() && parsedName!=null){// && preferredFlag){
121
                // do not ignore authorship for non-preferred names because they need
122
                // to be created for the determination history
123
                String nameCache = TaxonName.castAndDeproxy(parsedName).getNameCache();
124
                List<TaxonName> names = getNameService().findNamesByNameCache(nameCache, MatchMode.EXACT, null);
125
                if (!names.isEmpty()){
126
                     taxonName = getBestMatchingName(scientificName, new ArrayList<>(names), state);
127
                }
128
                if (taxonName == null && !names.isEmpty()){
129
                    taxonName = names.get(0);
130
                }
131

    
132
            } else {
133
                //search for existing names
134
                List<TaxonName> names = getNameService().listByTitle(TaxonName.class, scientificName, MatchMode.EXACT, null, null, null, null, null);
135
                taxonName = getBestMatchingName(scientificName, names, state);
136
                //still nothing found -> try with the atomised name full title cache
137
                if(taxonName==null && atomisedTaxonName!=null){
138
                    names = getNameService().listByTitle(TaxonName.class, atomisedTaxonName.getFullTitleCache(), MatchMode.EXACT, null, null, null, null, null);
139
                    taxonName = getBestMatchingName(atomisedTaxonName.getTitleCache(), names, state);
140
                    //still nothing found -> try with the atomised name title cache
141
                    if(taxonName==null){
142
                        names = getNameService().listByTitle(TaxonName.class, atomisedTaxonName.getTitleCache(), MatchMode.EXACT, null, null, null, null, null);
143
                        taxonName = getBestMatchingName(atomisedTaxonName.getTitleCache(), names, state);
144
                    }
145
                }
146

    
147
            }
148

    
149
        }
150

    
151
        if(taxonName==null && atomisedTaxonName!=null){
152
            taxonName = atomisedTaxonName;
153
            state.getReport().addName(taxonName);
154
            logger.info("Created new taxon name "+taxonName);
155
            if(taxonName.hasProblem()){
156
                state.getReport().addInfoMessage(String.format("Created %s with parsing problems", taxonName));
157
            }
158
            if(!atomisedTaxonName.getTitleCache().equals(scientificName)){
159
                state.getReport().addInfoMessage(String.format("Taxon %s was parsed as %s", scientificName, atomisedTaxonName.getTitleCache()));
160
            }
161
        }
162
        else if(taxonName==null){
163
            //create new taxon name
164

    
165
            if (state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNAFP)){
166
                taxonName = TaxonNameFactory.NewBotanicalInstance(rank);
167
            }else if (state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICZN)){
168
                taxonName = TaxonNameFactory.NewZoologicalInstance(rank);
169
            }else{
170
                taxonName = TaxonNameFactory.NewNonViralInstance(rank);
171
            }
172
            taxonName.setFullTitleCache(scientificName,true);
173
            taxonName.setTitleCache(scientificName, true);
174
            state.getReport().addName(taxonName);
175
            logger.info("Created new taxon name "+taxonName);
176
        }
177
        save(taxonName, state);
178
        return taxonName;
179
    }
180

    
181
	 protected TaxonName getBestMatchingName(String scientificName, java.util.Collection<TaxonName> names, STATE state){
182
	        Set<TaxonName> namesWithAcceptedTaxa = new HashSet<>();
183
	        List<TaxonName> namesWithAcceptedTaxaInClassification = new ArrayList<>();
184
	        for (TaxonName name : names) {
185
	            if(!name.getTaxa().isEmpty()){
186
	                Set<Taxon> taxa = name.getTaxa();
187
	                for (Taxon taxon:taxa){
188
	                    if (!taxon.getTaxonNodes().isEmpty()){
189
	                        //use only taxa included in a classification
190
	                        for (TaxonNode node:taxon.getTaxonNodes()){
191
	                            if (state.getClassification() != null && node.getClassification().equals(state.getClassification())){
192
	                                namesWithAcceptedTaxaInClassification.add(name);
193
	                            }else {
194
	                                namesWithAcceptedTaxa.add(name);
195
	                            }
196
	                        }
197

    
198
	                    }
199
	                }
200

    
201
	            }
202
	        }
203
	        String message = String.format("More than one taxon name was found for %s, maybe in other classifications!", scientificName);
204
	        //check for names with accepted taxa in classification
205
	        if(namesWithAcceptedTaxaInClassification.size()>0){
206
                if(namesWithAcceptedTaxaInClassification.size()>1){
207

    
208
                    state.getReport().addInfoMessage(message);
209
                    logger.warn(message);
210
                    return null;
211
                }
212
                return namesWithAcceptedTaxaInClassification.iterator().next();
213
            }
214
	      //check for any names with accepted taxa
215
	        if(namesWithAcceptedTaxa.size()>0){
216
	            if(namesWithAcceptedTaxa.size()>1){
217

    
218
	                state.getReport().addInfoMessage(message);
219
	                logger.warn(message);
220
	                return null;
221
	            }
222
	            return namesWithAcceptedTaxa.iterator().next();
223
	        }
224
	        //no names with accepted taxa found -> check accepted taxa of synonyms
225
	        List<Taxon> taxaFromSynonyms = new ArrayList<>();
226
	        for (TaxonName name : names) {
227
	            Set<TaxonBase> taxonBases = name.getTaxonBases();
228
	            for (TaxonBase taxonBase : taxonBases) {
229
	                if(taxonBase.isInstanceOf(Synonym.class)){
230
	                    Synonym synonym = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
231
	                    taxaFromSynonyms.add(synonym.getAcceptedTaxon());
232
	                }
233
	            }
234
	        }
235
	        if(taxaFromSynonyms.size()>0){
236
	            if(taxaFromSynonyms.size()>1){
237
	                state.getReport().addInfoMessage(message);
238
	                logger.warn(message);
239
	                return null;
240
	            }
241
	            return taxaFromSynonyms.iterator().next().getName();
242
	        }
243
	        //no accepted and no synonyms -> return one of the names and create a new taxon
244
	        if (names.isEmpty()){
245
	            return null;
246
	        }else{
247
	            return names.iterator().next();
248
	        }
249
	    }
250
	 /**
251
	     * Parse automatically the scientific name
252
	     * @param scientificName the scientific name to parse
253
	     * @param state the current import state
254
	     * @param report the import report
255
	     * @return a parsed name
256
	     */
257

    
258
	    protected TaxonName parseScientificName(String scientificName, STATE state, SpecimenImportReport report, Rank rank) {
259

    
260
	        NonViralNameParserImpl nvnpi = NonViralNameParserImpl.NewInstance();
261
	        TaxonName taxonName = null;
262
	        boolean problem = false;
263

    
264
	        if (logger.isDebugEnabled()){
265
	            logger.debug("parseScientificName " + state.getDataHolder().getNomenclatureCode().toString());
266
	        }
267

    
268
	        if (state.getDataHolder().getNomenclatureCode().toString().equals("Zoological") || state.getDataHolder().getNomenclatureCode().toString().contains("ICZN")) {
269
	            taxonName = (TaxonName)nvnpi.parseFullName(scientificName, NomenclaturalCode.ICZN, rank);
270
	            if (taxonName.hasProblem()) {
271
	                problem = true;
272
	            }
273
	        }
274
	        else if (state.getDataHolder().getNomenclatureCode().toString().equals("Botanical") || state.getDataHolder().getNomenclatureCode().toString().contains("ICBN")) {
275
	            taxonName = (TaxonName)nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNAFP, rank);
276
	            if (taxonName.hasProblem()) {
277
	                problem = true;
278
	            }
279
	        }
280
	        else if (state.getDataHolder().getNomenclatureCode().toString().equals("Bacterial") || state.getDataHolder().getNomenclatureCode().toString().contains("ICBN")) {
281
	            taxonName = (TaxonName)nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNB, rank);
282
	            if (taxonName.hasProblem()) {
283
	                problem = true;
284
	            }
285
	        }
286
	        else if (state.getDataHolder().getNomenclatureCode().toString().equals("Cultivar") || state.getDataHolder().getNomenclatureCode().toString().contains("ICNCP")) {
287
	            taxonName = (TaxonName)nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNCP, rank);
288
	            if (taxonName.hasProblem()) {
289
	                problem = true;
290
	            }
291
	        }
292
	        if (problem) {
293
	            String message = String.format("Parsing problems for %s", scientificName);
294
	            if(taxonName!=null){
295
	                for (ParserProblem parserProblem : taxonName.getParsingProblems()) {
296
	                    message += "\n\t- "+parserProblem;
297
	                }
298
	            }
299
	            report.addInfoMessage(message);
300
	            logger.info(message);
301
	        }
302
	        return taxonName;
303

    
304
	    }
305

    
306
	    /**
307
	     * Create the name without automatic parsing, either because it failed, or because the user deactivated it.
308
	     * The name is built upon the ABCD fields
309
	     * @param atomisedMap : the ABCD atomised fields
310
	     * @param fullName : the full scientific name
311
	     * @param state
312
	     * @return the corresponding Botanical or Zoological or... name
313
	     */
314
	    protected TaxonName setTaxonNameByType(
315
	            HashMap<String, String> atomisedMap, String fullName, STATE state) {
316
	        boolean problem = false;
317
	        if (logger.isDebugEnabled()){
318
	            logger.debug("settaxonnamebytype " + state.getDataHolder().getNomenclatureCode().toString());
319
	        }
320

    
321
	        if (state.getDataHolder().getNomenclatureCode().equals("Zoological") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICZN.getUuid())) {
322
	            TaxonName taxonName = TaxonNameFactory.NewZoologicalInstance(null);
323
	            taxonName.setFullTitleCache(fullName, true);
324
	            taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
325
	            taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
326
	            taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap,"SpeciesEpithet")));
327
	            taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap,"SubspeciesEpithet")));
328

    
329
	            if (taxonName.getGenusOrUninomial() != null){
330
	                taxonName.setRank(Rank.GENUS());
331
	            }
332

    
333
	            if (taxonName.getInfraGenericEpithet() != null){
334
	                taxonName.setRank(Rank.SUBGENUS());
335
	            }
336

    
337
	            if (taxonName.getSpecificEpithet() != null){
338
	                taxonName.setRank(Rank.SPECIES());
339
	            }
340

    
341
	            if (taxonName.getInfraSpecificEpithet() != null){
342
	                taxonName.setRank(Rank.SUBSPECIES());
343
	            }
344

    
345
	            Team team = null;
346
	            if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
347
	                team = Team.NewInstance();
348
	                team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
349
	            }
350
	            else {
351
	                if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
352
	                    team = Team.NewInstance();
353
	                    team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
354
	                }
355
	            }
356
	            if (team != null) {
357
	                taxonName.setBasionymAuthorship(team);
358
	            }
359
	            else {
360
	                if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
361
	                    taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
362
	                }
363
	                else if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
364
	                    taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamAndYear"));
365
	                }
366
	            }
367
	            if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
368
	                team = Team.NewInstance();
369
	                team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
370
	                taxonName.setCombinationAuthorship(team);
371
	            }
372
	            if (taxonName.hasProblem()) {
373
	                logger.info("pb ICZN");
374
	                problem = true;
375
	            }
376
	            else {
377
	                return taxonName;
378
	            }
379
	        }
380
	        else if (state.getDataHolder().getNomenclatureCode().equals("Botanical") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNAFP.getUuid())) {
381
	            TaxonName taxonName = parseScientificName(fullName, state, state.getReport(), null);
382
	            if (taxonName != null){
383
	                return taxonName;
384
	            }
385
	            else{
386
	                taxonName = TaxonNameFactory.NewBotanicalInstance(null);
387
	            }
388
	            taxonName.setFullTitleCache(fullName, true);
389
	            taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
390
	            taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "FirstEpithet")));
391
	            taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "InfraSpeEpithet")));
392
	            try {
393
	                taxonName.setRank(Rank.getRankByName(getFromMap(atomisedMap, "Rank")));
394
	            } catch (Exception e) {
395
	                if (taxonName.getInfraSpecificEpithet() != null){
396
	                    taxonName.setRank(Rank.SUBSPECIES());
397
	                }
398
	                else if (taxonName.getSpecificEpithet() != null){
399
	                    taxonName.setRank(Rank.SPECIES());
400
	                }
401
	                else if (taxonName.getInfraGenericEpithet() != null){
402
	                    taxonName.setRank(Rank.SUBGENUS());
403
	                }
404
	                else if (taxonName.getGenusOrUninomial() != null){
405
	                    taxonName.setRank(Rank.GENUS());
406
	                }
407
	            }
408
	            Team team = null;
409
	            if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
410
	                team = Team.NewInstance();
411
	                team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
412
	                taxonName.setBasionymAuthorship(team);
413
	            }
414
	            if (getFromMap(atomisedMap, "AuthorTeam") != null) {
415
	                team = Team.NewInstance();
416
	                team.setTitleCache(getFromMap(atomisedMap, "AuthorTeam"), true);
417
	                taxonName.setCombinationAuthorship(team);
418
	            }
419
	            if (team == null) {
420
	                if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
421
	                    taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
422
	                }
423
	                else if (getFromMap(atomisedMap, "AuthorTeam") != null) {
424
	                    taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeam"));
425
	                }
426
	            }
427
	            if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
428
	                team = Team.NewInstance();
429
	                team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
430
	                taxonName.setCombinationAuthorship(team);
431
	            }
432
	            if (taxonName.hasProblem()) {
433
	                logger.info("pb ICBN");
434
	                problem = true;
435
	            }
436
	            else {
437
	                return taxonName;
438
	            }
439
	        }
440
	        else if (state.getDataHolder().getNomenclatureCode().equals("Bacterial") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNB.getUuid())) {
441
	            TaxonName taxonName = TaxonNameFactory.NewBacterialInstance(null);
442
	            taxonName.setFullTitleCache(fullName, true);
443
	            taxonName.setGenusOrUninomial(getFromMap(atomisedMap, "Genus"));
444
	            taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
445
	            taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "Species")));
446
	            taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "SubspeciesEpithet")));
447

    
448
	            if (taxonName.getGenusOrUninomial() != null){
449
	                taxonName.setRank(Rank.GENUS());
450
	            }
451
	            else if (taxonName.getInfraGenericEpithet() != null){
452
	                taxonName.setRank(Rank.SUBGENUS());
453
	            }
454
	            else if (taxonName.getSpecificEpithet() != null){
455
	                taxonName.setRank(Rank.SPECIES());
456
	            }
457
	            else if (taxonName.getInfraSpecificEpithet() != null){
458
	                taxonName.setRank(Rank.SUBSPECIES());
459
	            }
460

    
461
	            if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
462
	                Team team = Team.NewInstance();
463
	                team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
464
	                taxonName.setCombinationAuthorship(team);
465
	            }
466
	            if (getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear") != null) {
467
	                Team team = Team.NewInstance();
468
	                team.setTitleCache(getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear"), true);
469
	                taxonName.setBasionymAuthorship(team);
470
	            }
471
	            if (taxonName.hasProblem()) {
472
	                logger.info("pb ICNB");
473
	                problem = true;
474
	            }
475
	            else {
476
	                return taxonName;
477
	            }
478
	        }
479
	        else if (state.getDataHolder().getNomenclatureCode().equals("Cultivar")) {
480
	            TaxonName taxonName = TaxonNameFactory.NewCultivarInstance(null);
481

    
482
	            if (taxonName.hasProblem()) {
483
	                logger.info("pb ICNCP");
484
	                problem = true;
485
	            }
486
	            else {
487
	                return taxonName;
488
	            }
489
	            return taxonName;
490
	        }
491

    
492
	        if (problem) {
493
	            logger.info("Problem im setTaxonNameByType ");
494
	            TaxonName taxonName = TaxonNameFactory.NewNonViralInstance(null);
495
	            taxonName.setFullTitleCache(fullName, true);
496
	            return taxonName;
497
	        }
498
	        TaxonName tn = TaxonNameFactory.NewNonViralInstance(null);
499
	        return tn;
500
	    }
501

    
502
	    /**
503
	     * Get a formated string from a hashmap
504
	     * @param atomisedMap
505
	     * @param key
506
	     * @return
507
	     */
508
	    private String getFromMap(HashMap<String, String> atomisedMap, String key) {
509
	        String value = null;
510
	        if (atomisedMap.containsKey(key)) {
511
	            value = atomisedMap.get(key);
512
	        }
513

    
514
	        try {
515
	            if (value != null && key.matches(".*Year.*")) {
516
	                value = value.trim();
517
	                if (value.matches("[a-z A-Z ]*[0-9]{4}$")) {
518
	                    String tmp = value.split("[0-9]{4}$")[0];
519
	                    int year = Integer.parseInt(value.split(tmp)[1]);
520
	                    if (year >= 1752) {
521
	                        value = tmp;
522
	                    }
523
	                    else {
524
	                        value = null;
525
	                    }
526
	                }
527
	                else {
528
	                    value = null;
529
	                }
530
	            }
531
	        }
532
	        catch (Exception e) {
533
	            value = null;
534
	        }
535
	        return value;
536
	    }
537

    
538
	    /**
539
	     * Very fast and dirty implementation to allow handling of transient objects as described in
540
	     * https://dev.e-taxonomy.eu/trac/ticket/3726
541
	     *
542
	     * Not yet complete.
543
	     *
544
	     * @param cdmBase
545
	     * @param state
546
	     */
547
	    protected UUID save(CdmBase cdmBase, SpecimenImportStateBase state) {
548
	        ICdmRepository cdmRepository = state.getConfig().getCdmAppController();
549
	        if (cdmRepository == null){
550
	            cdmRepository = this;
551
	        }
552

    
553
	        if (cdmBase.isInstanceOf(LanguageString.class)){
554
	            return cdmRepository.getTermService().saveLanguageData(CdmBase.deproxy(cdmBase, LanguageString.class));
555
	        }else if (cdmBase.isInstanceOf(SpecimenOrObservationBase.class)){
556
	            return cdmRepository.getOccurrenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, SpecimenOrObservationBase.class));
557
	        }else if (cdmBase.isInstanceOf(Reference.class)){
558
	            return cdmRepository.getReferenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, Reference.class));
559
	        }else if (cdmBase.isInstanceOf(Classification.class)){
560
	            return cdmRepository.getClassificationService().saveOrUpdate(CdmBase.deproxy(cdmBase, Classification.class));
561
	        }else if (cdmBase.isInstanceOf(AgentBase.class)){
562
	            return cdmRepository.getAgentService().saveOrUpdate(CdmBase.deproxy(cdmBase, AgentBase.class));
563
	        }else if (cdmBase.isInstanceOf(Collection.class)){
564
	            return cdmRepository.getCollectionService().saveOrUpdate(CdmBase.deproxy(cdmBase, Collection.class));
565
	        }else if (cdmBase.isInstanceOf(DescriptionBase.class)){
566
	            return cdmRepository.getDescriptionService().saveOrUpdate(CdmBase.deproxy(cdmBase, DescriptionBase.class));
567
	        }else if (cdmBase.isInstanceOf(TaxonBase.class)){
568
	            return cdmRepository.getTaxonService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonBase.class));
569
	        }else if (cdmBase.isInstanceOf(TaxonName.class)){
570
	            return cdmRepository.getNameService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonName.class));
571
	        }else if (cdmBase.isInstanceOf(TaxonNode.class)){
572
                return cdmRepository.getTaxonNodeService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonNode.class));
573
            }else{
574
	            throw new IllegalArgumentException("Class not supported in save method: " + CdmBase.deproxy(cdmBase, CdmBase.class).getClass().getSimpleName());
575
	        }
576

    
577
	    }
578

    
579

    
580
	    protected SpecimenOrObservationBase findExistingSpecimen(String unitId, SpecimenImportStateBase state){
581
	        ICdmRepository cdmAppController = state.getConfig().getCdmAppController();
582
	        if(cdmAppController==null){
583
	            cdmAppController = this;
584
	        }
585
	        FindOccurrencesConfigurator config = new FindOccurrencesConfigurator();
586
	        config.setSignificantIdentifier(unitId);
587
	        Pager<SpecimenOrObservationBase> existingSpecimens = cdmAppController.getOccurrenceService().findByTitle(config);
588
	        if(!existingSpecimens.getRecords().isEmpty()){
589
	            if(existingSpecimens.getRecords().size()==1){
590
	                return existingSpecimens.getRecords().iterator().next();
591
	            }
592
	        }
593
	        return null;
594
	    }
595

    
596
	    protected abstract void importAssociatedUnits(STATE state, Object item, DerivedUnitFacade derivedUnitFacade);
597

    
598
	    /**
599
	     * getFacade : get the DerivedUnitFacade based on the recordBasis
600
	     * @param state
601
	     *
602
	     * @return DerivedUnitFacade
603
	     */
604
	    protected DerivedUnitFacade getFacade(STATE state) {
605
	        if (logger.isDebugEnabled()){
606
	            logger.info("getFacade()");
607
	        }
608
	        SpecimenOrObservationType type = null;
609

    
610
	        // create specimen
611
	        if (NB((state.getDataHolder().getRecordBasis())) != null) {
612
	            if (state.getDataHolder().getRecordBasis().toLowerCase().startsWith("s") || state.getDataHolder().getRecordBasis().toLowerCase().indexOf("specimen")>-1) {// specimen
613
	                type = SpecimenOrObservationType.PreservedSpecimen;
614
	            }
615
	            if (state.getDataHolder().getRecordBasis().toLowerCase().startsWith("o") ||state.getDataHolder().getRecordBasis().toLowerCase().indexOf("observation")>-1 ) {
616
	                type = SpecimenOrObservationType.Observation;
617
	            }
618
	            if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("fossil")>-1){
619
	                type = SpecimenOrObservationType.Fossil;
620
	            }
621
	            if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("living")>-1) {
622
	                type = SpecimenOrObservationType.LivingSpecimen;
623
	            }
624
	            if (type == null) {
625
	                logger.info("The basis of record does not seem to be known: " + state.getDataHolder().getRecordBasis());
626
	                type = SpecimenOrObservationType.DerivedUnit;
627
	            }
628
	            // TODO fossils?
629
	        } else {
630
	            logger.info("The basis of record is null");
631
	            type = SpecimenOrObservationType.DerivedUnit;
632
	        }
633
	        DerivedUnitFacade derivedUnitFacade = DerivedUnitFacade.NewInstance(type);
634
	        return derivedUnitFacade;
635
	    }
636

    
637
	    /**
638
	     * Look if the Institution does already exist
639
	     * @param institutionCode: a string with the institutioncode
640
	     * @param config : the configurator
641
	     * @return the Institution (existing or new)
642
	     */
643
	    protected Institution getInstitution(String institutionCode, STATE state) {
644
	        SpecimenImportConfiguratorBase config = state.getConfig();
645
	        Institution institution=null;
646
	        institution = (Institution)state.institutions.get(institutionCode);
647
	        if (institution != null){
648
	            return institution;
649
	        }
650
	        List<Institution> institutions;
651
	        try {
652
	            institutions = getAgentService().searchInstitutionByCode(institutionCode);
653

    
654
	        } catch (Exception e) {
655
	            institutions = new ArrayList<Institution>();
656
	            logger.warn(e);
657
	        }
658
	        if (institutions.size() > 0 && config.isReuseExistingMetaData()) {
659
	            for (Institution institut:institutions){
660
	                try{
661
	                    if (institut.getCode().equalsIgnoreCase(institutionCode)) {
662
	                        institution=institut;
663
	                        break;
664
	                    }
665
	                }catch(Exception e){logger.warn("no institution code in the db");}
666
	            }
667
	        }
668
	        if (logger.isDebugEnabled()){
669
	            if(institution !=null) {
670
	                logger.info("getinstitution " + institution.toString());
671
	            }
672
	        }
673
	        if (institution == null){
674
	            // create institution
675
	            institution = Institution.NewInstance();
676
	            institution.setCode(institutionCode);
677
	            institution.setTitleCache(institutionCode, true);
678
	            UUID uuid = save(institution, state);
679
	        }
680

    
681

    
682
	        state.institutions.put(institutionCode, institution);
683
	        return institution;
684
	    }
685

    
686
	    /**
687
	     * Look if the Collection does already exist
688
	     * @param collectionCode
689
	     * @param collectionCode: a string
690
	     * @param config : the configurator
691
	     * @return the Collection (existing or new)
692
	     */
693
	    protected Collection getCollection(Institution institution, String collectionCode, STATE state) {
694
	        SpecimenImportConfiguratorBase config = state.getConfig();
695
	        Collection collection = null;
696
	        List<Collection> collections;
697
	        collection = (Collection) state.collections.get(collectionCode);
698
	        if (collection != null){
699
	            return collection;
700
	        }
701
	        try {
702
	            collections = getCollectionService().searchByCode(collectionCode);
703
	        } catch (Exception e) {
704
	            collections = new ArrayList<Collection>();
705
	        }
706
	        if (collections.size() > 0 && config.isReuseExistingMetaData()) {
707
	            for (Collection coll:collections){
708
	                if (coll.getCode() != null && coll.getInstitute() != null
709
	                        && coll.getCode().equalsIgnoreCase(collectionCode) && coll.getInstitute().equals(institution)) {
710
	                    collection = coll;
711
	                    break;
712
	                }
713
	            }
714
	        }
715

    
716
	        if(collection == null){
717
	            collection =Collection.NewInstance();
718
	            collection.setCode(collectionCode);
719
	            collection.setInstitute(institution);
720
	            collection.setTitleCache(collectionCode);
721
	            UUID uuid = save(collection, state);
722
	        }
723

    
724

    
725

    
726
	        state.collections.put(collectionCode, collection);
727

    
728
	        return collection;
729
	    }
730

    
731
	    /**
732
	     * @param reference
733
	     * @param citationDetail
734
	     * @return
735
	     */
736
	    //FIXME this method is highly critical, because
737
	    //  * it will have serious performance and memory problems with large databases
738
	    //        (databases may easily have >1 Mio source records)
739
	    //  * it does not make sense to search for existing sources and then clone them
740
	    //    we need to search for existing references instead and use them (if exist)
741
	    //    for our new source.
742
	    protected IdentifiableSource getIdentifiableSource(Reference reference, String citationDetail) {
743

    
744
	      /*  List<IdentifiableSource> issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
745

    
746

    
747
	        if (reference != null){
748
	            try {
749
	                for (OriginalSourceBase<?> osb: issTmp){
750
	                    if (osb.getCitation() != null && osb.getCitation().getTitleCache().equalsIgnoreCase(reference.getTitleCache())){
751
	                        String osbDetail = osb.getCitationMicroReference();
752
	                        if ((StringUtils.isBlank(osbDetail) && StringUtils.isBlank(citationDetail))
753
	                                || (osbDetail != null && osbDetail.equalsIgnoreCase(citationDetail)) ) {
754
//	                            System.out.println("REFERENCE FOUND RETURN EXISTING SOURCE");
755
	                            return (IdentifiableSource) osb.clone();
756
	                        }
757
	                    }
758
	                }
759
	            } catch (CloneNotSupportedException e) {
760
	                throw new RuntimeException(e);
761
	            } catch (Exception e1){
762
	                e1.printStackTrace();
763
	            }
764
	        }
765
	*/
766
	        IdentifiableSource sour = IdentifiableSource.NewInstance(OriginalSourceType.Import,null,null, reference,citationDetail);
767
	        return sour;
768
	    }
769

    
770
	    /**
771
	     * Add the hierarchy for a Taxon(add higher taxa)
772
	     * @param classification
773
	     * @param taxon: a taxon to add as a node
774
	     * @param state: the ABCD import state
775
	     */
776
	    protected void addParentTaxon(Taxon taxon, STATE state, boolean preferredFlag, Classification classification){
777
	        INonViralName  nvname = taxon.getName();
778
	        Rank rank = nvname.getRank();
779
	        Taxon genus =null;
780
	        Taxon subgenus =null;
781
	        Taxon species = null;
782
	        Taxon subspecies = null;
783
	        Taxon parent = null;
784
	        if(rank!=null){
785
	            if (rank.isLower(Rank.GENUS() )){
786
	                String genusOrUninomial = nvname.getGenusOrUninomial();
787
	                TaxonName taxonName = getOrCreateTaxonName(genusOrUninomial, Rank.GENUS(), preferredFlag, state, -1);
788
	                genus = getOrCreateTaxonForName(taxonName, state);
789
	                if (genus == null){
790
	                    logger.debug("The genus should not be null " + taxonName);
791
	                }
792
	                if (preferredFlag) {
793
	                    parent = linkParentChildNode(null, genus, classification, state);
794
	                }
795

    
796
	            }
797
	            if (rank.isLower(Rank.SUBGENUS())){
798
	                String prefix = nvname.getGenusOrUninomial();
799
	                String name = nvname.getInfraGenericEpithet();
800
	                if (name != null){
801
	                    TaxonName taxonName = getOrCreateTaxonName(prefix+" "+name, Rank.SUBGENUS(), preferredFlag, state, -1);
802
	                    subgenus = getOrCreateTaxonForName(taxonName, state);
803
	                    if (preferredFlag) {
804
	                        parent = linkParentChildNode(genus, subgenus, classification, state);
805
	                    }            }
806
	            }
807
	            if (rank.isLower(Rank.SPECIES())){
808
	                if (subgenus!=null){
809
	                    String prefix = nvname.getGenusOrUninomial();
810
	                    String name = nvname.getInfraGenericEpithet();
811
	                    String spe = nvname.getSpecificEpithet();
812
	                    if (spe != null){
813
	                        TaxonName taxonName = getOrCreateTaxonName(prefix+" "+name+" "+spe, Rank.SPECIES(), preferredFlag, state, -1);
814
	                        species = getOrCreateTaxonForName(taxonName, state);
815
	                        if (preferredFlag) {
816
	                            parent = linkParentChildNode(subgenus, species, classification, state);
817
	                        }
818
	                    }
819
	                }
820
	                else{
821
	                    String prefix = nvname.getGenusOrUninomial();
822
	                    String name = nvname.getSpecificEpithet();
823
	                    if (name != null){
824
	                        TaxonName taxonName = getOrCreateTaxonName(prefix+" "+name, Rank.SPECIES(), preferredFlag, state, -1);
825
	                        species = getOrCreateTaxonForName(taxonName, state);
826
	                        if (preferredFlag) {
827
	                            parent = linkParentChildNode(genus, species, classification, state);
828
	                        }
829
	                    }
830
	                }
831
	            }
832
	            if (rank.isLower(Rank.INFRASPECIES())){
833
	                TaxonName taxonName = getOrCreateTaxonName(nvname.getFullTitleCache(), Rank.SUBSPECIES(), preferredFlag, state, -1);
834
	                subspecies = getOrCreateTaxonForName(taxonName, state);
835
	                if (preferredFlag) {
836
	                    parent = linkParentChildNode(species, subspecies, classification, state);
837
	                }
838
	            }
839
	        }
840
	        if (preferredFlag && parent!=taxon ) {
841
	            linkParentChildNode(parent, taxon, classification, state);
842
	        }
843
	    }
844

    
845
	    /**
846
	     * Link a parent to a child and save it in the current classification
847
	     * @param parent: the higher Taxon
848
	     * @param child : the lower (or current) Taxon
849
	     * return the Taxon from the new created Node
850
	     * @param classification
851
	     * @param state
852
	     */
853
	    protected Taxon linkParentChildNode(Taxon parent, Taxon child, Classification classification, STATE state) {
854
	        TaxonNode node =null;
855
	        if (parent != null) {
856
	            parent = (Taxon) getTaxonService().find(parent.getUuid());
857
	            child = (Taxon) getTaxonService().find(child.getUuid());
858
	            //here we do not have to check if the taxon nodes already exists
859
	            //this is done by classification.addParentChild()
860
	            //do not add child node if it already exists
861
	            if(hasTaxonNodeInClassification(child, classification)){
862
	                return child;
863
	            }
864
	            else{
865
	                node = classification.addParentChild(parent, child, state.getRef(), "");
866
	                save(node, state);
867
	            }
868
	        }
869
	        else {
870
	            if (child == null){
871
	                logger.debug("The child should not be null!");
872
	            }
873
	            child = (Taxon) getTaxonService().find(child.getUuid());
874
	            //do not add child node if it already exists
875
	            if(hasTaxonNodeInClassification(child, classification)){
876
	                return child;
877
	            }
878
	            else{
879
	                node = classification.addChildTaxon(child, state.getRef(), null);
880
	                save(node, state);
881
	            }
882
	        }
883
	        if(node!=null){
884
	            state.getReport().addTaxonNode(node);
885
	            return node.getTaxon();
886
	        }
887
	        String message = "Could not create taxon node for " +child;
888
	        state.getReport().addInfoMessage(message);
889
	        logger.warn(message);
890
	        return null;
891
	    }
892

    
893
	    protected Taxon getOrCreateTaxonForName(TaxonName taxonName, STATE state){
894
	        if (taxonName != null){
895
    	        Set<Taxon> acceptedTaxa = taxonName.getTaxa();
896
    	        if(acceptedTaxa.size()>0){
897
    	            Taxon firstAcceptedTaxon = acceptedTaxa.iterator().next();
898
    	            if(acceptedTaxa.size()>1){
899
    	                String message = "More than one accepted taxon was found for taxon name: "
900
    	                        + taxonName.getTitleCache() + "!\n" + firstAcceptedTaxon + "was chosen for "+state.getDerivedUnitBase();
901
    	                state.getReport().addInfoMessage(message);
902
    	                logger.warn(message);
903
    	            }
904
    	            else{
905
    	                return firstAcceptedTaxon;
906
    	            }
907
    	        }
908
    	        else{
909
    	            Set<TaxonBase> taxonAndSynonyms = taxonName.getTaxonBases();
910
    	            for (TaxonBase taxonBase : taxonAndSynonyms) {
911
    	                if(taxonBase.isInstanceOf(Synonym.class)){
912
    	                    Synonym synonym = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
913
    	                    Taxon acceptedTaxonOfSynonym = synonym.getAcceptedTaxon();
914
    	                    if(acceptedTaxonOfSynonym == null){
915
    	                        String message = "No accepted taxon could be found for taxon name: "
916
    	                                + taxonName.getTitleCache()
917
    	                                + "!";
918
    	                        state.getReport().addInfoMessage(message);
919
    	                        logger.warn(message);
920
    	                    }
921
    	                    else{
922
    	                        return acceptedTaxonOfSynonym;
923
    	                    }
924
    	                }
925
    	            }
926
    	        }
927
    	        Taxon taxon = Taxon.NewInstance(taxonName, state.getRef());
928
    	        save(taxon, state);
929
    	        state.getReport().addTaxon(taxon);
930
    	        logger.info("Created new taxon "+ taxon);
931
    	        return taxon;
932
	        }
933
	        return null;
934

    
935
	    }
936

    
937
	    private boolean hasTaxonNodeInClassification(Taxon taxon, Classification classification){
938
	        if(taxon.getTaxonNodes()!=null){
939
	            for (TaxonNode node : taxon.getTaxonNodes()){
940
	                if(node.getClassification().equals(classification)){
941
	                    return true;
942
	                }
943
	            }
944
	        }
945
	        return false;
946
	    }
947

    
948
	    /**
949
	     * HandleIdentifications : get the scientific names present in the ABCD
950
	     * document and store link them with the observation/specimen data
951
	     * @param state: the current ABCD import state
952
	     * @param derivedUnitFacade : the current derivedunitfacade
953
	     */
954
	    protected void handleIdentifications(STATE state, DerivedUnitFacade derivedUnitFacade) {
955
	       SpecimenImportConfiguratorBase config = state.getConfig();
956

    
957

    
958
	        String scientificName = "";
959
	        boolean preferredFlag = false;
960

    
961
	        if (state.getDataHolder().getNomenclatureCode() == ""){
962
	            if (config.getNomenclaturalCode() != null){
963
	                if (config.getNomenclaturalCode() != null){
964
	                    state.getDataHolder().setNomenclatureCode(config.getNomenclaturalCode().toString());
965

    
966
	                }
967
	            }
968
	        }
969

    
970
	        for (int i = 0; i < state.getDataHolder().getIdentificationList().size(); i++) {
971
	            Identification identification = state.getDataHolder().getIdentificationList().get(i);
972
	            scientificName = identification.getScientificName().replaceAll(" et ", " & ");
973

    
974
	            String preferred = identification.getPreferred();
975
	            preferredFlag = false;
976
	            if (preferred != null){
977
    	            if (preferred.equals("1") || preferred.toLowerCase().indexOf("true") != -1 || state.getDataHolder().getIdentificationList().size()==1) {
978
    	                preferredFlag = true;
979
    	            }
980

    
981
	            }
982
	            if (identification.getCode() != null){
983
    	            if (identification.getCode().indexOf(':') != -1) {
984
    	                state.getDataHolder().setNomenclatureCode(identification.getCode().split(COLON)[1]);
985
    	            }
986
    	            else{
987
    	                state.getDataHolder().setNomenclatureCode(identification.getCode());
988
    	            }
989
	            }
990
	            TaxonName taxonName = getOrCreateTaxonName(scientificName, null, preferredFlag, state, i);
991
	            Taxon taxon = getOrCreateTaxonForName(taxonName, state);
992
	            addTaxonNode(taxon, state,preferredFlag);
993
	            linkDeterminationEvent(state, taxon, preferredFlag, derivedUnitFacade, identification.getIdentifier(), identification.getDate());
994
	        }
995
	    }
996

    
997
	    /**
998
	     * @param taxon : a taxon to add as a node
999
	     * @param state : the ABCD import state
1000
	     */
1001
	    protected void addTaxonNode(Taxon taxon, STATE state, boolean preferredFlag) {
1002
	        SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
1003
	        logger.info("link taxon to a taxonNode "+taxon.getTitleCache());
1004
	        //only add nodes if not already existing in current classification or default classification
1005

    
1006
	        //check if node exists in current classification
1007
	        //NOTE: we cannot use hasTaxonNodeInClassification() here because we are first creating it here
1008
	        if (!existsInClassification(taxon,state.getClassification(), state)){
1009
	            if(config.isMoveNewTaxaToDefaultClassification()){
1010
	                //check if node exists in default classification
1011
	                if (!existsInClassification(taxon, state.getDefaultClassification(), state)){
1012
	                    addParentTaxon(taxon, state, preferredFlag, state.getDefaultClassification());
1013
	                }
1014
	            }
1015
	            else {
1016
	                //add non-existing taxon to current classification
1017
	                addParentTaxon(taxon, state, preferredFlag, state.getClassification());
1018
	            }
1019
	        }
1020
	    }
1021

    
1022

    
1023
	    private boolean existsInClassification(Taxon taxon, Classification classification, STATE state){
1024
	        boolean exist = false;
1025
	        ICdmRepository cdmAppController = state.getConfig().getCdmAppController();
1026
            if(cdmAppController==null){
1027
                cdmAppController = this;
1028
            }
1029
            if (classification != null){
1030
                if (!taxon.getTaxonNodes().isEmpty()){
1031
                    for (TaxonNode node:taxon.getTaxonNodes()){
1032
                        if (node.getClassification().equals(classification)){
1033
                            return true;
1034
                        }
1035
                    }
1036
                }
1037
// we do not need this because we already searched for taxa in db in the previous steps
1038
//    	        List<UuidAndTitleCache<TaxonNode>> uuidAndTitleCacheOfAllTaxa = cdmAppController.getClassificationService().getTaxonNodeUuidAndTitleCacheOfAcceptedTaxaByClassification(classification.getUuid());
1039
//    	        if (uuidAndTitleCacheOfAllTaxa != null){
1040
//        	        for (UuidAndTitleCache p : uuidAndTitleCacheOfAllTaxa){
1041
//        	            try{
1042
//        	                if(p.getTitleCache().equals(taxon.getTitleCache())) {
1043
//        	                    exist = true;
1044
//        	                }
1045
//        	            }
1046
//        	            catch(Exception e){
1047
//        	                logger.warn("TaxonNode doesn't seem to have a taxon");
1048
//        	            }
1049
//        	        }
1050
//    	        }
1051
            }
1052
	        return exist;
1053
	    }
1054

    
1055
	    /**
1056
	     * join DeterminationEvent to the Taxon Object
1057
	     * @param state : the ABCD import state
1058
	     * @param taxon: the current Taxon
1059
	     * @param preferredFlag :if the current name is preferred
1060
	     * @param derivedFacade : the derived Unit Facade
1061
	     */
1062
	    @SuppressWarnings("rawtypes")
1063
        protected void linkDeterminationEvent(STATE state, Taxon taxon, boolean preferredFlag,  DerivedUnitFacade derivedFacade, String identifierStr, String dateStr) {
1064
	        SpecimenImportConfiguratorBase config = state.getConfig();
1065
	        if (logger.isDebugEnabled()){
1066
	            logger.info("start linkdetermination with taxon:" + taxon.getUuid()+", "+taxon);
1067
	        }
1068

    
1069
	        DeterminationEvent determinationEvent = DeterminationEvent.NewInstance();
1070
	        determinationEvent.setTaxonName(taxon.getName());
1071
	        determinationEvent.setPreferredFlag(preferredFlag);
1072

    
1073

    
1074
	        determinationEvent.setIdentifiedUnit(state.getDerivedUnitBase());
1075
	        if (state.getPersonStore().get(identifierStr) != null){
1076
	            determinationEvent.setActor((AgentBase)state.getPersonStore().get(identifierStr));
1077
	        } else if (identifierStr != null){
1078
	            Person identifier = Person.NewTitledInstance(identifierStr);
1079
	            determinationEvent.setActor(identifier);
1080
	        }
1081
	        if (dateStr != null){
1082
	            determinationEvent.setTimeperiod(TimePeriodParser.parseString(dateStr));
1083
	        }
1084
	        state.getDerivedUnitBase().addDetermination(determinationEvent);
1085

    
1086
	        if (logger.isDebugEnabled()){
1087
	            logger.debug("NB TYPES INFO: "+ state.getDataHolder().getStatusList().size());
1088
	        }
1089
	        for (SpecimenTypeDesignationStatus specimenTypeDesignationstatus : state.getDataHolder().getStatusList()) {
1090
	            if (specimenTypeDesignationstatus != null) {
1091
	                if (logger.isDebugEnabled()){
1092
	                    logger.debug("specimenTypeDesignationstatus :"+ specimenTypeDesignationstatus);
1093
	                }
1094

    
1095
	                ICdmRepository cdmAppController = config.getCdmAppController();
1096
	                if(cdmAppController == null){
1097
	                    cdmAppController = this;
1098
	                }
1099
	                specimenTypeDesignationstatus = (SpecimenTypeDesignationStatus) cdmAppController.getTermService().find(specimenTypeDesignationstatus.getUuid());
1100
	                //Designation
1101
	                TaxonName name = taxon.getName();
1102
	                SpecimenTypeDesignation designation = SpecimenTypeDesignation.NewInstance();
1103

    
1104
	                designation.setTypeStatus(specimenTypeDesignationstatus);
1105
	                designation.setTypeSpecimen(state.getDerivedUnitBase());
1106
	                name.addTypeDesignation(designation, true);
1107
	            }
1108
	        }
1109

    
1110
	        for (String[] fullReference : state.getDataHolder().getReferenceList()) {
1111

    
1112

    
1113
	            String strReference=fullReference[0];
1114
	            String citationDetail = fullReference[1];
1115
	            String citationURL = fullReference[2];
1116
	            List<Reference> references = getReferenceService().listByTitle(Reference.class, "strReference", MatchMode.EXACT, null, null, null, null, null);
1117

    
1118
	            if (!references.isEmpty()){
1119
	                Reference reference = null;
1120
	                for (Reference refe: references) {
1121
	                    if (refe.getTitleCache().equalsIgnoreCase(strReference)) {
1122
	                        reference =refe;
1123
	                        break;
1124
	                    }
1125
	                }
1126
	                if (reference ==null){
1127
	                    reference = ReferenceFactory.newGeneric();
1128
	                    reference.setTitleCache(strReference, true);
1129
	                    save(reference, state);
1130
	                }
1131
	                determinationEvent.addReference(reference);
1132
	            }
1133
	        }
1134
	        save(state.getDerivedUnitBase(), state);
1135

    
1136
	        if (config.isAddIndividualsAssociationsSuchAsSpecimenAndObservations() && preferredFlag) {
1137
	            //do not add IndividualsAssociation to non-preferred taxa
1138
	            if (logger.isDebugEnabled()){
1139
	                logger.debug("isDoCreateIndividualsAssociations");
1140
	            }
1141

    
1142
	            makeIndividualsAssociation(state, taxon, determinationEvent);
1143

    
1144
	            save(state.getDerivedUnitBase(), state);
1145
	        }
1146
	    }
1147

    
1148
	    /**
1149
	     * create and link each association (specimen, observation..) to the accepted taxon
1150
	     * @param state : the ABCD import state
1151
	     * @param taxon: the current Taxon
1152
	     * @param determinationEvent:the determinationevent
1153
	     */
1154
	    protected void makeIndividualsAssociation(STATE state, Taxon taxon, DeterminationEvent determinationEvent) {
1155
	        SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
1156
	        SpecimenUserInteraction sui = config.getSpecimenUserInteraction();
1157

    
1158
	        if (logger.isDebugEnabled()){
1159
	            logger.info("MAKE INDIVIDUALS ASSOCIATION");
1160
	        }
1161

    
1162
	        TaxonDescription taxonDescription = null;
1163
	        Set<TaxonDescription> descriptions= taxon.getDescriptions();
1164
	       if (!descriptions.isEmpty()){ taxonDescription = descriptions.iterator().next();}
1165

    
1166
//	            for (TaxonDescription description : descriptions){
1167
//	                Set<IdentifiableSource> sources =  new HashSet<>();
1168
//	                sources.addAll(description.getTaxon().getSources());
1169
//	                sources.addAll(description.getSources());
1170
//	                for (IdentifiableSource source:sources){
1171
//	                    if(state.getRef().equals(source.getCitation())) {
1172
//	                        taxonDescription = description;
1173
//	                    }
1174
//	                }
1175
//	            }
1176
	      //  }
1177
	        if (taxonDescription == null){
1178
	            taxonDescription = TaxonDescription.NewInstance(taxon, false);
1179
	            if(sourceNotLinkedToElement(taxonDescription,state.getRef(),null)) {
1180
	                taxonDescription.addSource(OriginalSourceType.Import, null, null, state.getRef(), null);
1181
	            }
1182
	            state.setDescriptionGroup(taxonDescription);
1183
	            taxon.addDescription(taxonDescription);
1184
	        }
1185

    
1186
	        //PREPARE REFERENCE QUESTIONS
1187

    
1188
	        Map<String,OriginalSourceBase<?>> sourceMap = new HashMap<String, OriginalSourceBase<?>>();
1189

    
1190
	        List<IdentifiableSource> issTmp = new ArrayList<>();//getCommonService().list(IdentifiableSource.class, null, null, null, null);
1191
	        List<DescriptionElementSource> issTmp2 = new ArrayList<>();//getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1192

    
1193
	        Set<OriginalSourceBase> osbSet = new HashSet<OriginalSourceBase>();
1194
	        if(issTmp2!=null) {
1195
	            osbSet.addAll(issTmp2);
1196
	        }
1197
	        if(issTmp!=null) {
1198
	            osbSet.addAll(issTmp);
1199
	        }
1200

    
1201

    
1202
	        addToSourceMap(sourceMap, osbSet);
1203

    
1204
//	        if (((Abcd206ImportConfigurator) state.getConfig()).isInteractWithUser()){
1205
//	            List<OriginalSourceBase<?>> res = null;
1206
//	            if(!state.isDescriptionSourcesSet()){
1207
//	                res = sui.askForSource(sourceMap, "the description group ("+taxon+")",
1208
//	                        "The current reference is "+state.getRef().getTitleCache(),getReferenceService(), state.getDataHolder().docSources);
1209
//	                state.setDescriptionRefs(res);
1210
//	                state.setDescriptionSourcesSet(true);
1211
//	            }
1212
//	            else{
1213
//	                res=state.getDescriptionRefs();
1214
//	            }
1215
//	            if(res !=null) {
1216
//	                for (OriginalSourceBase<?> sour:res){
1217
//	                    if(sour.isInstanceOf(IdentifiableSource.class)){
1218
//	                        try {
1219
//	                            if(sourceNotLinkedToElement(taxonDescription,sour)) {
1220
//	                                taxonDescription.addSource((IdentifiableSource)sour.clone());
1221
//	                            }
1222
//	                        } catch (CloneNotSupportedException e) {
1223
//	                            logger.warn("no cloning?");
1224
//	                        }
1225
//	                    }else{
1226
//	                        if(sourceNotLinkedToElement(taxonDescription,sour)) {
1227
//	                            taxonDescription.addSource(OriginalSourceType.Import,null, null, sour.getCitation(),sour.getCitationMicroReference());
1228
//	                        }
1229
//	                    }
1230
//	                }
1231
//	            }
1232
//	        }
1233
//	        else {
1234
	            if(sourceNotLinkedToElement(taxonDescription,state.getRef(),null)) {
1235
	                taxonDescription.addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1236
	            }
1237
//	        }
1238
	        state.setDescriptionGroup(taxonDescription);
1239

    
1240
	        IndividualsAssociation indAssociation = IndividualsAssociation.NewInstance();
1241
	        Feature feature = makeFeature(state.getDerivedUnitBase());
1242
	        indAssociation.setAssociatedSpecimenOrObservation(state.getDerivedUnitBase());
1243
	        indAssociation.setFeature(feature);
1244
//
1245
//	        if (((Abcd206ImportConfigurator) state.getConfig()).isInteractWithUser()){
1246
//	            sourceMap = new HashMap<String, OriginalSourceBase<?>>();
1247
//
1248
//	            issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
1249
//	            issTmp2 = getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1250
//
1251
//	            osbSet = new HashSet<OriginalSourceBase>();
1252
//	            if(issTmp2!=null) {
1253
//	                osbSet.addAll(issTmp2);
1254
//	            }
1255
//	            if(issTmp!=null) {
1256
//	                osbSet.addAll(issTmp);
1257
//	            }
1258
//
1259
//
1260
//	            addToSourceMap(sourceMap, osbSet);
1261
//
1262
//	            List<OriginalSourceBase<?>> sources =null;
1263
//	            if(!state.isAssociationSourcesSet()) {
1264
//	                sources = sui.askForSource(sourceMap,  "descriptive element (association) ",taxon.toString(),
1265
//	                        getReferenceService(),state.getDataHolder().getDocSources());
1266
//	                state.setAssociationRefs(sources);
1267
//	                state.setAssociationSourcesSet(true);
1268
//	            }
1269
//	            else{
1270
//	                sources=state.getAssociationRefs();
1271
//	            }
1272
//	            if(sources !=null) {
1273
//	                for (OriginalSourceBase<?> source: sources) {
1274
//	                    if(source !=null) {
1275
//	                        if(source.isInstanceOf(DescriptionElementSource.class)){
1276
//	                            try {
1277
//	                                if(sourceNotLinkedToElement(indAssociation,source)) {
1278
//	                                    indAssociation.addSource((DescriptionElementSource)source.clone());
1279
//	                                }
1280
//	                            } catch (CloneNotSupportedException e) {
1281
//	                                logger.warn("clone forbidden?");
1282
//	                            }
1283
//	                        }else{
1284
//	                            if(sourceNotLinkedToElement(indAssociation,source)) {
1285
//	                                indAssociation.addSource(OriginalSourceType.Import,null, null, source.getCitation(),source.getCitationMicroReference());
1286
//	                            }
1287
//	                            try {
1288
//	                                if(sourceNotLinkedToElement(state.getDerivedUnitBase(), source)) {
1289
//	                                    state.getDerivedUnitBase().addSource((IdentifiableSource) source.clone());
1290
//	                                }
1291
//	                            } catch (CloneNotSupportedException e) {
1292
//	                                // TODO Auto-generated catch block
1293
//	                                e.printStackTrace();
1294
//	                            }
1295
//	                        }
1296
//
1297
//	                    }
1298
//	                }
1299
//	            }
1300
//	        }else {
1301
	            if(sourceNotLinkedToElement(indAssociation,state.getRef(),null)) {
1302
	                indAssociation.addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1303
	            }
1304
	            if(sourceNotLinkedToElement(state.getDerivedUnitBase(), state.getRef(),null)) {
1305
	                state.getDerivedUnitBase().addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1306
	            }
1307
	            for (Reference citation : determinationEvent.getReferences()) {
1308
	                if(sourceNotLinkedToElement(indAssociation,citation,null))
1309
	                {
1310
	                    indAssociation.addSource(DescriptionElementSource.NewInstance(OriginalSourceType.Import, null, null, citation, null));
1311
	                }
1312
	                if(sourceNotLinkedToElement(state.getDerivedUnitBase(), state.getRef(),null)) {
1313
	                    state.getDerivedUnitBase().addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1314
	                }
1315
	            }
1316
	   //     }
1317

    
1318
	        taxonDescription.addElement(indAssociation);
1319

    
1320
	        save(taxonDescription, state);
1321
	        save(taxon, state);
1322
	        state.getReport().addDerivate(state.getDerivedUnitBase(), config);
1323
	        state.getReport().addIndividualAssociation(taxon, state.getDataHolder().getUnitID(), state.getDerivedUnitBase());
1324
	    }
1325

    
1326
	    /**
1327
	     * @param derivedUnitBase2
1328
	     * @param ref2
1329
	     * @param object
1330
	     * @return
1331
	     */
1332
	    private boolean sourceNotLinkedToElement(DerivedUnit derivedUnitBase2, Reference b, String d) {
1333
	        Set<IdentifiableSource> linkedSources = derivedUnitBase2.getSources();
1334
	        for (IdentifiableSource is:linkedSources){
1335
	            Reference a = is.getCitation();
1336
	            String c = is.getCitationMicroReference();
1337

    
1338
	            boolean refMatch=false;
1339
	            boolean microMatch=false;
1340

    
1341
	            try{
1342
	                if (a==null && b==null) {
1343
	                    refMatch=true;
1344
	                }
1345
	                if (a!=null && b!=null) {
1346
	                    if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1347
	                        refMatch=true;
1348
	                    }
1349
	                }
1350
	            }catch(Exception e){}
1351

    
1352

    
1353
	            try{
1354
	                if (c==null && d==null) {
1355
	                    microMatch=true;
1356
	                }
1357
	                if(c!=null && d!=null) {
1358
	                    if(c.equalsIgnoreCase(d)) {
1359
	                        microMatch=true;
1360
	                    }
1361
	                }
1362
	            }
1363
	            catch(Exception e){}
1364

    
1365
	            if (microMatch && refMatch) {
1366
	                return false;
1367
	            }
1368

    
1369

    
1370
	        }
1371
	        return true;
1372
	    }
1373

    
1374
	    private <T extends OriginalSourceBase<?>> boolean  sourceNotLinkedToElement(ISourceable<T> sourcable, Reference reference, String microReference) {
1375
	        Set<T> linkedSources = sourcable.getSources();
1376
	        for (T is:linkedSources){
1377
	            Reference unitReference = is.getCitation();
1378
	            String unitMicroReference = is.getCitationMicroReference();
1379

    
1380
	            boolean refMatch=false;
1381
	            boolean microMatch=false;
1382

    
1383
	            try{
1384
	                if (unitReference==null && reference==null) {
1385
	                    refMatch=true;
1386
	                }
1387
	                if (unitReference!=null && reference!=null) {
1388
	                    if (unitReference.getTitleCache().equalsIgnoreCase(reference.getTitleCache())) {
1389
	                        refMatch=true;
1390
	                    }
1391
	                }
1392
	            }catch(Exception e){}
1393

    
1394
	            try{
1395
	                if (unitMicroReference==null && microReference==null) {
1396
	                    microMatch=true;
1397
	                }
1398
	                if(unitMicroReference!=null && microReference!=null) {
1399
	                    if(unitMicroReference.equalsIgnoreCase(microReference)) {
1400
	                        microMatch=true;
1401
	                    }
1402
	                }
1403
	            }
1404
	            catch(Exception e){}
1405

    
1406
	            if (microMatch && refMatch) {
1407
	                return false;
1408
	            }
1409
	        }
1410
	        return true;
1411
	    }
1412

    
1413
	    /**
1414
	     * look for the Feature object (FieldObs, Specimen,...)
1415
	     * @param unit : a specimen or obersvation base
1416
	     * @return the corresponding Feature
1417
	     */
1418
	    private Feature makeFeature(SpecimenOrObservationBase<?> unit) {
1419
	        SpecimenOrObservationType type = unit.getRecordBasis();
1420

    
1421

    
1422

    
1423
	        if (type.isFeatureObservation()){
1424
	            return Feature.OBSERVATION();
1425
	        }else if (type.isFeatureSpecimen()){
1426
	            return Feature.SPECIMEN();
1427
	        }else if (type == SpecimenOrObservationType.DerivedUnit){
1428
	            return Feature.OBSERVATION();
1429
	            //            return getFeature("Specimen or observation");
1430
	        }else{
1431
	            String message = "Unhandled record basis '%s' for defining individuals association feature type. Use default.";
1432
	            logger.warn(String.format(message, type.getMessage()));
1433
	            return Feature.OBSERVATION();
1434
	            //            return getFeature("Specimen or observation");
1435

    
1436
	        }
1437
	    }
1438

    
1439

    
1440
	    /**
1441
	     * @param sourceMap
1442
	     * @param osbSet
1443
	     */
1444
	    protected void addToSourceMap(Map<String, OriginalSourceBase<?>> sourceMap, Set<OriginalSourceBase> osbSet) {
1445
	        for( OriginalSourceBase<?> osb:osbSet) {
1446
	            if(osb.getCitation()!=null && osb.getCitationMicroReference() !=null  && !osb.getCitationMicroReference().isEmpty()) {
1447
	                try{
1448
	                    sourceMap.put(osb.getCitation().getTitleCache()+ "---"+osb.getCitationMicroReference(),osb);
1449
	                }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1450
	            } else if(osb.getCitation()!=null){
1451
	                try{
1452
	                    sourceMap.put(osb.getCitation().getTitleCache(),osb);
1453
	                }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1454
	            }
1455
	        }
1456
	    }
1457

    
1458

    
1459
}
(2-2/7)