Project

General

Profile

Download (64.6 KB) Statistics
| Branch: | Tag: | Revision:
1
/**
2
* Copyright (C) 2007 EDIT
3
* European Distributed Institute of Taxonomy
4
* http://www.e-taxonomy.eu
5
*
6
* The contents of this file are subject to the Mozilla Public License Version 1.1
7
* See LICENSE.TXT at the top of this package for the full license terms.
8
*/
9

    
10
package eu.etaxonomy.cdm.io.specimen;
11

    
12

    
13
import java.util.ArrayList;
14
import java.util.HashMap;
15
import java.util.HashSet;
16
import java.util.List;
17
import java.util.Map;
18
import java.util.Set;
19
import java.util.UUID;
20

    
21
import org.apache.log4j.Logger;
22

    
23
import eu.etaxonomy.cdm.api.application.ICdmRepository;
24
import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
25
import eu.etaxonomy.cdm.api.service.config.FindOccurrencesConfigurator;
26
import eu.etaxonomy.cdm.api.service.pager.Pager;
27
import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
28
import eu.etaxonomy.cdm.io.common.CdmImportBase;
29
import eu.etaxonomy.cdm.io.common.IImportConfigurator;
30
import eu.etaxonomy.cdm.io.specimen.abcd206.in.Identification;
31
import eu.etaxonomy.cdm.io.specimen.abcd206.in.SpecimenImportReport;
32
import eu.etaxonomy.cdm.model.agent.AgentBase;
33
import eu.etaxonomy.cdm.model.agent.Institution;
34
import eu.etaxonomy.cdm.model.agent.Person;
35
import eu.etaxonomy.cdm.model.agent.Team;
36
import eu.etaxonomy.cdm.model.common.CdmBase;
37
import eu.etaxonomy.cdm.model.common.ISourceable;
38
import eu.etaxonomy.cdm.model.common.IdentifiableSource;
39
import eu.etaxonomy.cdm.model.common.LanguageString;
40
import eu.etaxonomy.cdm.model.common.OriginalSourceBase;
41
import eu.etaxonomy.cdm.model.common.OriginalSourceType;
42
import eu.etaxonomy.cdm.model.description.DescriptionBase;
43
import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
44
import eu.etaxonomy.cdm.model.description.Feature;
45
import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
46
import eu.etaxonomy.cdm.model.description.TaxonDescription;
47
import eu.etaxonomy.cdm.model.name.INonViralName;
48
import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
49
import eu.etaxonomy.cdm.model.name.Rank;
50
import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
51
import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
52
import eu.etaxonomy.cdm.model.name.TaxonName;
53
import eu.etaxonomy.cdm.model.name.TaxonNameFactory;
54
import eu.etaxonomy.cdm.model.occurrence.Collection;
55
import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
56
import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
57
import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
58
import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationType;
59
import eu.etaxonomy.cdm.model.reference.Reference;
60
import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
61
import eu.etaxonomy.cdm.model.taxon.Classification;
62
import eu.etaxonomy.cdm.model.taxon.Synonym;
63
import eu.etaxonomy.cdm.model.taxon.Taxon;
64
import eu.etaxonomy.cdm.model.taxon.TaxonBase;
65
import eu.etaxonomy.cdm.model.taxon.TaxonNode;
66
import eu.etaxonomy.cdm.persistence.query.MatchMode;
67
import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
68
import eu.etaxonomy.cdm.strategy.parser.ParserProblem;
69
import eu.etaxonomy.cdm.strategy.parser.TimePeriodParser;
70

    
71

    
72
/**
73
 * @author p.kelbert
74
 * @created 20.10.2008
75
 */
76
public abstract class SpecimenImportBase<CONFIG extends IImportConfigurator, STATE extends SpecimenImportStateBase>
77
        extends CdmImportBase<CONFIG, STATE> {
78

    
79
    private static final long serialVersionUID = 4423065367998125678L;
80
    private static final Logger logger = Logger.getLogger(SpecimenImportBase.class);
81

    
82
	protected static final UUID SPECIMEN_SCAN_TERM = UUID.fromString("acda15be-c0e2-4ea8-8783-b9b0c4ad7f03");
83

    
84
	private static final String COLON = ":";
85

    
86

    
87
	@Override
88
    protected abstract void doInvoke(STATE state);
89

    
90
	/**
91
	 * Handle a single unit
92
	 * @param state
93
	 * @param item
94
	 */
95
	protected abstract void handleSingleUnit(STATE state, Object item) ;
96

    
97

    
98

    
99
	protected TaxonName getOrCreateTaxonName(String scientificName, Rank rank, boolean preferredFlag, STATE state, int unitIndexInAbcdFile){
100
	    TaxonName taxonName = null;
101
        SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
102

    
103
        //check atomised name data for rank
104
        //new name will be created
105
        TaxonName atomisedTaxonName = null;
106
        if (rank==null && unitIndexInAbcdFile>=0 && ((state.getDataHolder().getAtomisedIdentificationList() != null && !state.getDataHolder().getAtomisedIdentificationList().isEmpty())|| state.getDataHolder().getAtomisedIdentificationList().size() > 0)) {
107
            atomisedTaxonName = setTaxonNameByType(state.getDataHolder().getAtomisedIdentificationList().get(unitIndexInAbcdFile), scientificName, state);
108
            if(atomisedTaxonName!=null){
109
                rank = atomisedTaxonName.getRank();
110
            }
111
        }
112
        if(config.isReuseExistingTaxaWhenPossible()){
113
            TaxonName parsedName = atomisedTaxonName;
114
            if(parsedName==null){
115

    
116
                parsedName = parseScientificName(scientificName, state, state.getReport(), rank);
117

    
118
            }
119
            atomisedTaxonName = parsedName;
120
            if(config.isIgnoreAuthorship() && parsedName!=null){// && preferredFlag){
121
                // do not ignore authorship for non-preferred names because they need
122
                // to be created for the determination history
123
                String nameCache = TaxonName.castAndDeproxy(parsedName).getNameCache();
124
                List<TaxonName> names = getNameService().findNamesByNameCache(nameCache, MatchMode.EXACT, null);
125
                if (!names.isEmpty()){
126
                     taxonName = getBestMatchingName(scientificName, new ArrayList<>(names), state);
127
                }
128
                if (taxonName == null && !names.isEmpty()){
129
                    taxonName = names.get(0);
130
                }
131

    
132
            } else {
133
                //search for existing names
134
                List<TaxonName> names = getNameService().listByTitle(TaxonName.class, scientificName, MatchMode.EXACT, null, null, null, null, null);
135
                taxonName = getBestMatchingName(scientificName, names, state);
136
                //still nothing found -> try with the atomised name full title cache
137
                if(taxonName==null && atomisedTaxonName!=null){
138
                    names = getNameService().listByTitle(TaxonName.class, atomisedTaxonName.getFullTitleCache(), MatchMode.EXACT, null, null, null, null, null);
139
                    taxonName = getBestMatchingName(atomisedTaxonName.getTitleCache(), names, state);
140
                    //still nothing found -> try with the atomised name title cache
141
                    if(taxonName==null){
142
                        names = getNameService().listByTitle(TaxonName.class, atomisedTaxonName.getTitleCache(), MatchMode.EXACT, null, null, null, null, null);
143
                        taxonName = getBestMatchingName(atomisedTaxonName.getTitleCache(), names, state);
144
                    }
145
                }
146

    
147
            }
148

    
149
        }
150

    
151
        if(taxonName==null && atomisedTaxonName!=null){
152
            taxonName = atomisedTaxonName;
153
            state.getReport().addName(taxonName);
154
            logger.info("Created new taxon name "+taxonName);
155
            if(taxonName.hasProblem()){
156
                state.getReport().addInfoMessage(String.format("Created %s with parsing problems", taxonName));
157
            }
158
            if(!atomisedTaxonName.getTitleCache().equals(scientificName)){
159
                state.getReport().addInfoMessage(String.format("Taxon %s was parsed as %s", scientificName, atomisedTaxonName.getTitleCache()));
160
            }
161
        }
162
        else if(taxonName==null){
163
            //create new taxon name
164

    
165
            if (state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNAFP)){
166
                taxonName = TaxonNameFactory.NewBotanicalInstance(rank);
167
            }else if (state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICZN)){
168
                taxonName = TaxonNameFactory.NewZoologicalInstance(rank);
169
            }else{
170
                taxonName = TaxonNameFactory.NewNonViralInstance(rank);
171
            }
172
            taxonName.setFullTitleCache(scientificName,true);
173
            taxonName.setTitleCache(scientificName, true);
174
            state.getReport().addName(taxonName);
175
            logger.info("Created new taxon name "+taxonName);
176
        }
177
        save(taxonName, state);
178
        return taxonName;
179
    }
180

    
181
	 protected TaxonName getBestMatchingName(String scientificName, java.util.Collection<TaxonName> names, STATE state){
182
	        Set<TaxonName> namesWithAcceptedTaxa = new HashSet<>();
183
	        List<TaxonName> namesWithAcceptedTaxaInClassification = new ArrayList<>();
184
	        for (TaxonName name : names) {
185
	            if(!name.getTaxa().isEmpty()){
186
	                Set<Taxon> taxa = name.getTaxa();
187
	                for (Taxon taxon:taxa){
188
	                    if (!taxon.getTaxonNodes().isEmpty()){
189
	                        //use only taxa included in a classification
190
	                        for (TaxonNode node:taxon.getTaxonNodes()){
191
	                            if (state.getClassification() != null && node.getClassification().equals(state.getClassification())){
192
	                                namesWithAcceptedTaxaInClassification.add(name);
193
	                            }else {
194
	                                namesWithAcceptedTaxa.add(name);
195
	                            }
196
	                        }
197

    
198
	                    }
199
	                }
200

    
201
	            }
202
	        }
203
	        String message = String.format("More than one taxon name was found for %s, maybe in other classifications!", scientificName);
204
	        //check for names with accepted taxa in classification
205
	        if(namesWithAcceptedTaxaInClassification.size()>0){
206
                if(namesWithAcceptedTaxaInClassification.size()>1){
207

    
208
                    state.getReport().addInfoMessage(message);
209
                    logger.warn(message);
210
                    return null;
211
                }
212
                return namesWithAcceptedTaxaInClassification.iterator().next();
213
            }
214
	      //check for any names with accepted taxa
215
	        if(namesWithAcceptedTaxa.size()>0){
216
	            if(namesWithAcceptedTaxa.size()>1){
217

    
218
	                state.getReport().addInfoMessage(message);
219
	                logger.warn(message);
220
	                return null;
221
	            }
222
	            return namesWithAcceptedTaxa.iterator().next();
223
	        }
224
	        //no names with accepted taxa found -> check accepted taxa of synonyms
225
	        List<Taxon> taxaFromSynonyms = new ArrayList<>();
226
	        for (TaxonName name : names) {
227
	            Set<TaxonBase> taxonBases = name.getTaxonBases();
228
	            for (TaxonBase taxonBase : taxonBases) {
229
	                if(taxonBase.isInstanceOf(Synonym.class)){
230
	                    Synonym synonym = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
231
	                    taxaFromSynonyms.add(synonym.getAcceptedTaxon());
232
	                }
233
	            }
234
	        }
235
	        if(taxaFromSynonyms.size()>0){
236
	            if(taxaFromSynonyms.size()>1){
237
	                state.getReport().addInfoMessage(message);
238
	                logger.warn(message);
239
	                return null;
240
	            }
241
	            return taxaFromSynonyms.iterator().next().getName();
242
	        }
243
	        //no accepted and no synonyms -> return one of the names and create a new taxon
244
	        if (names.isEmpty()){
245
	            return null;
246
	        }else{
247
	            return names.iterator().next();
248
	        }
249
	    }
250
	 /**
251
	     * Parse automatically the scientific name
252
	     * @param scientificName the scientific name to parse
253
	     * @param state the current import state
254
	     * @param report the import report
255
	     * @return a parsed name
256
	     */
257

    
258
	    protected TaxonName parseScientificName(String scientificName, STATE state, SpecimenImportReport report, Rank rank) {
259

    
260
	        NonViralNameParserImpl nvnpi = NonViralNameParserImpl.NewInstance();
261
	        TaxonName taxonName = null;
262
	        boolean problem = false;
263

    
264
	        if (logger.isDebugEnabled()){
265
	            logger.debug("parseScientificName " + state.getDataHolder().getNomenclatureCode().toString());
266
	        }
267

    
268
	        if (state.getDataHolder().getNomenclatureCode().toString().equals("Zoological") || state.getDataHolder().getNomenclatureCode().toString().contains("ICZN")) {
269
	            taxonName = (TaxonName)nvnpi.parseFullName(scientificName, NomenclaturalCode.ICZN, rank);
270
	            if (taxonName.hasProblem()) {
271
	                problem = true;
272
	            }
273
	        }
274
	        else if (state.getDataHolder().getNomenclatureCode().toString().equals("Botanical") || state.getDataHolder().getNomenclatureCode().toString().contains("ICBN")) {
275
	            taxonName = (TaxonName)nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNAFP, rank);
276
	            if (taxonName.hasProblem()) {
277
	                problem = true;
278
	            }
279
	        }
280
	        else if (state.getDataHolder().getNomenclatureCode().toString().equals("Bacterial") || state.getDataHolder().getNomenclatureCode().toString().contains("ICBN")) {
281
	            taxonName = (TaxonName)nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNB, rank);
282
	            if (taxonName.hasProblem()) {
283
	                problem = true;
284
	            }
285
	        }
286
	        else if (state.getDataHolder().getNomenclatureCode().toString().equals("Cultivar") || state.getDataHolder().getNomenclatureCode().toString().contains("ICNCP")) {
287
	            taxonName = (TaxonName)nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNCP, rank);
288
	            if (taxonName.hasProblem()) {
289
	                problem = true;
290
	            }
291
	        }
292
	        if (problem) {
293
	            String message = String.format("Parsing problems for %s", scientificName);
294
	            if(taxonName!=null){
295
	                for (ParserProblem parserProblem : taxonName.getParsingProblems()) {
296
	                    message += "\n\t- "+parserProblem;
297
	                }
298
	            }
299
	            report.addInfoMessage(message);
300
	            logger.info(message);
301
	        }
302
	        return taxonName;
303

    
304
	    }
305

    
306
	    /**
307
	     * Create the name without automatic parsing, either because it failed, or because the user deactivated it.
308
	     * The name is built upon the ABCD fields
309
	     * @param atomisedMap : the ABCD atomised fields
310
	     * @param fullName : the full scientific name
311
	     * @param state
312
	     * @return the corresponding Botanical or Zoological or... name
313
	     */
314
	    protected TaxonName setTaxonNameByType(
315
	            HashMap<String, String> atomisedMap, String fullName, STATE state) {
316
	        boolean problem = false;
317
	        if (logger.isDebugEnabled()){
318
	            logger.debug("settaxonnamebytype " + state.getDataHolder().getNomenclatureCode().toString());
319
	        }
320

    
321
	        if (state.getDataHolder().getNomenclatureCode().equals("Zoological") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICZN.getUuid())) {
322
	            TaxonName taxonName = TaxonNameFactory.NewZoologicalInstance(null);
323
	            taxonName.setFullTitleCache(fullName, true);
324
	            taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
325
	            taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
326
	            taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap,"SpeciesEpithet")));
327
	            taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap,"SubspeciesEpithet")));
328

    
329
	            if (taxonName.getGenusOrUninomial() != null){
330
	                taxonName.setRank(Rank.GENUS());
331
	            }
332

    
333
	            if (taxonName.getInfraGenericEpithet() != null){
334
	                taxonName.setRank(Rank.SUBGENUS());
335
	            }
336

    
337
	            if (taxonName.getSpecificEpithet() != null){
338
	                taxonName.setRank(Rank.SPECIES());
339
	            }
340

    
341
	            if (taxonName.getInfraSpecificEpithet() != null){
342
	                taxonName.setRank(Rank.SUBSPECIES());
343
	            }
344

    
345
	            Team team = null;
346
	            if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
347
	                team = Team.NewInstance();
348
	                team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
349
	            }
350
	            else {
351
	                if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
352
	                    team = Team.NewInstance();
353
	                    team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
354
	                }
355
	            }
356
	            if (team != null) {
357
	                taxonName.setBasionymAuthorship(team);
358
	            }
359
	            else {
360
	                if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
361
	                    taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
362
	                }
363
	                else if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
364
	                    taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamAndYear"));
365
	                }
366
	            }
367
	            if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
368
	                team = Team.NewInstance();
369
	                team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
370
	                taxonName.setCombinationAuthorship(team);
371
	            }
372
	            if (taxonName.hasProblem()) {
373
	                logger.info("pb ICZN");
374
	                problem = true;
375
	            }
376
	            else {
377
	                return taxonName;
378
	            }
379
	        }
380
	        else if (state.getDataHolder().getNomenclatureCode().equals("Botanical") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNAFP.getUuid())) {
381
	            TaxonName taxonName = parseScientificName(fullName, state, state.getReport(), null);
382
	            if (taxonName != null){
383
	                return taxonName;
384
	            }
385
	            else{
386
	                taxonName = TaxonNameFactory.NewBotanicalInstance(null);
387
	            }
388
	            taxonName.setFullTitleCache(fullName, true);
389
	            taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
390
	            taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "FirstEpithet")));
391
	            taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "InfraSpeEpithet")));
392
	            try {
393
	                taxonName.setRank(Rank.getRankByName(getFromMap(atomisedMap, "Rank")));
394
	            } catch (Exception e) {
395
	                if (taxonName.getInfraSpecificEpithet() != null){
396
	                    taxonName.setRank(Rank.SUBSPECIES());
397
	                }
398
	                else if (taxonName.getSpecificEpithet() != null){
399
	                    taxonName.setRank(Rank.SPECIES());
400
	                }
401
	                else if (taxonName.getInfraGenericEpithet() != null){
402
	                    taxonName.setRank(Rank.SUBGENUS());
403
	                }
404
	                else if (taxonName.getGenusOrUninomial() != null){
405
	                    taxonName.setRank(Rank.GENUS());
406
	                }
407
	            }
408
	            Team team = null;
409
	            if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
410
	                team = Team.NewInstance();
411
	                team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
412
	                taxonName.setBasionymAuthorship(team);
413
	            }
414
	            if (getFromMap(atomisedMap, "AuthorTeam") != null) {
415
	                team = Team.NewInstance();
416
	                team.setTitleCache(getFromMap(atomisedMap, "AuthorTeam"), true);
417
	                taxonName.setCombinationAuthorship(team);
418
	            }
419
	            if (team == null) {
420
	                if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
421
	                    taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
422
	                }
423
	                else if (getFromMap(atomisedMap, "AuthorTeam") != null) {
424
	                    taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeam"));
425
	                }
426
	            }
427
	            if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
428
	                team = Team.NewInstance();
429
	                team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
430
	                taxonName.setCombinationAuthorship(team);
431
	            }
432
	            if (taxonName.hasProblem()) {
433
	                logger.info("pb ICBN");
434
	                problem = true;
435
	            }
436
	            else {
437
	                return taxonName;
438
	            }
439
	        }
440
	        else if (state.getDataHolder().getNomenclatureCode().equals("Bacterial") || state.getDataHolder().getNomenclatureCode().equals(NomenclaturalCode.ICNB.getUuid())) {
441
	            TaxonName taxonName = TaxonNameFactory.NewBacterialInstance(null);
442
	            taxonName.setFullTitleCache(fullName, true);
443
	            taxonName.setGenusOrUninomial(getFromMap(atomisedMap, "Genus"));
444
	            taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
445
	            taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "Species")));
446
	            taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "SubspeciesEpithet")));
447

    
448
	            if (taxonName.getGenusOrUninomial() != null){
449
	                taxonName.setRank(Rank.GENUS());
450
	            }
451
	            else if (taxonName.getInfraGenericEpithet() != null){
452
	                taxonName.setRank(Rank.SUBGENUS());
453
	            }
454
	            else if (taxonName.getSpecificEpithet() != null){
455
	                taxonName.setRank(Rank.SPECIES());
456
	            }
457
	            else if (taxonName.getInfraSpecificEpithet() != null){
458
	                taxonName.setRank(Rank.SUBSPECIES());
459
	            }
460

    
461
	            if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
462
	                Team team = Team.NewInstance();
463
	                team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
464
	                taxonName.setCombinationAuthorship(team);
465
	            }
466
	            if (getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear") != null) {
467
	                Team team = Team.NewInstance();
468
	                team.setTitleCache(getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear"), true);
469
	                taxonName.setBasionymAuthorship(team);
470
	            }
471
	            if (taxonName.hasProblem()) {
472
	                logger.info("pb ICNB");
473
	                problem = true;
474
	            }
475
	            else {
476
	                return taxonName;
477
	            }
478
	        }
479
	        else if (state.getDataHolder().getNomenclatureCode().equals("Cultivar")) {
480
	            TaxonName taxonName = TaxonNameFactory.NewCultivarInstance(null);
481

    
482
	            if (taxonName.hasProblem()) {
483
	                logger.info("pb ICNCP");
484
	                problem = true;
485
	            }
486
	            else {
487
	                return taxonName;
488
	            }
489
	            return taxonName;
490
	        }
491

    
492
	        if (problem) {
493
	            logger.info("Problem im setTaxonNameByType ");
494
	            TaxonName taxonName = TaxonNameFactory.NewNonViralInstance(null);
495
	            taxonName.setFullTitleCache(fullName, true);
496
	            return taxonName;
497
	        }
498
	        TaxonName tn = TaxonNameFactory.NewNonViralInstance(null);
499
	        return tn;
500
	    }
501

    
502
	    /**
503
	     * Get a formated string from a hashmap
504
	     * @param atomisedMap
505
	     * @param key
506
	     * @return
507
	     */
508
	    private String getFromMap(HashMap<String, String> atomisedMap, String key) {
509
	        String value = null;
510
	        if (atomisedMap.containsKey(key)) {
511
	            value = atomisedMap.get(key);
512
	        }
513

    
514
	        try {
515
	            if (value != null && key.matches(".*Year.*")) {
516
	                value = value.trim();
517
	                if (value.matches("[a-z A-Z ]*[0-9]{4}$")) {
518
	                    String tmp = value.split("[0-9]{4}$")[0];
519
	                    int year = Integer.parseInt(value.split(tmp)[1]);
520
	                    if (year >= 1752) {
521
	                        value = tmp;
522
	                    }
523
	                    else {
524
	                        value = null;
525
	                    }
526
	                }
527
	                else {
528
	                    value = null;
529
	                }
530
	            }
531
	        }
532
	        catch (Exception e) {
533
	            value = null;
534
	        }
535
	        return value;
536
	    }
537

    
538
	    /**
539
	     * Very fast and dirty implementation to allow handling of transient objects as described in
540
	     * https://dev.e-taxonomy.eu/trac/ticket/3726
541
	     *
542
	     * Not yet complete.
543
	     *
544
	     * @param cdmBase
545
	     * @param state
546
	     */
547
	    protected UUID save(CdmBase cdmBase, SpecimenImportStateBase state) {
548
	        ICdmRepository cdmRepository = state.getConfig().getCdmAppController();
549
	        if (cdmRepository == null){
550
	            cdmRepository = this;
551
	        }
552

    
553
	        if (cdmBase.isInstanceOf(LanguageString.class)){
554
	            return cdmRepository.getTermService().saveLanguageData(CdmBase.deproxy(cdmBase, LanguageString.class));
555
	        }else if (cdmBase.isInstanceOf(SpecimenOrObservationBase.class)){
556
	            SpecimenOrObservationBase specimen = CdmBase.deproxy(cdmBase, SpecimenOrObservationBase.class);
557

    
558
	            return cdmRepository.getOccurrenceService().saveOrUpdate(specimen);
559
	        }else if (cdmBase.isInstanceOf(Reference.class)){
560
	            return cdmRepository.getReferenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, Reference.class));
561
	        }else if (cdmBase.isInstanceOf(Classification.class)){
562
	            return cdmRepository.getClassificationService().saveOrUpdate(CdmBase.deproxy(cdmBase, Classification.class));
563
	        }else if (cdmBase.isInstanceOf(AgentBase.class)){
564
	            return cdmRepository.getAgentService().saveOrUpdate(CdmBase.deproxy(cdmBase, AgentBase.class));
565
	        }else if (cdmBase.isInstanceOf(Collection.class)){
566
	            return cdmRepository.getCollectionService().saveOrUpdate(CdmBase.deproxy(cdmBase, Collection.class));
567
	        }else if (cdmBase.isInstanceOf(DescriptionBase.class)){
568
	            DescriptionBase description = CdmBase.deproxy(cdmBase, DescriptionBase.class);
569

    
570
	            return cdmRepository.getDescriptionService().saveOrUpdate(description);
571
	        }else if (cdmBase.isInstanceOf(TaxonBase.class)){
572
	            return cdmRepository.getTaxonService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonBase.class));
573
	        }else if (cdmBase.isInstanceOf(TaxonName.class)){
574
	            return cdmRepository.getNameService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonName.class));
575
	        }else if (cdmBase.isInstanceOf(TaxonNode.class)){
576
                return cdmRepository.getTaxonNodeService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonNode.class));
577
            }else{
578
	            throw new IllegalArgumentException("Class not supported in save method: " + CdmBase.deproxy(cdmBase, CdmBase.class).getClass().getSimpleName());
579
	        }
580

    
581
	    }
582

    
583

    
584
	    protected SpecimenOrObservationBase findExistingSpecimen(String unitId, SpecimenImportStateBase state){
585
	        ICdmRepository cdmAppController = state.getConfig().getCdmAppController();
586
	        if(cdmAppController==null){
587
	            cdmAppController = this;
588
	        }
589
	        FindOccurrencesConfigurator config = new FindOccurrencesConfigurator();
590
	        config.setSignificantIdentifier(unitId);
591
	        Pager<SpecimenOrObservationBase> existingSpecimens = cdmAppController.getOccurrenceService().findByTitle(config);
592
	        if(!existingSpecimens.getRecords().isEmpty()){
593
	            if(existingSpecimens.getRecords().size()==1){
594
	                return existingSpecimens.getRecords().iterator().next();
595
	            }
596
	        }
597
	        return null;
598
	    }
599

    
600
	    protected abstract void importAssociatedUnits(STATE state, Object item, DerivedUnitFacade derivedUnitFacade);
601

    
602
	    /**
603
	     * getFacade : get the DerivedUnitFacade based on the recordBasis
604
	     * @param state
605
	     *
606
	     * @return DerivedUnitFacade
607
	     */
608
	    protected DerivedUnitFacade getFacade(STATE state) {
609
	        if (logger.isDebugEnabled()){
610
	            logger.info("getFacade()");
611
	        }
612
	        SpecimenOrObservationType type = null;
613

    
614
	        // create specimen
615
	        if (NB((state.getDataHolder().getRecordBasis())) != null) {
616
	            if (state.getDataHolder().getRecordBasis().toLowerCase().startsWith("s") || state.getDataHolder().getRecordBasis().toLowerCase().indexOf("specimen")>-1) {// specimen
617
	                type = SpecimenOrObservationType.PreservedSpecimen;
618
	            }
619
	            if (state.getDataHolder().getRecordBasis().toLowerCase().startsWith("o") ||state.getDataHolder().getRecordBasis().toLowerCase().indexOf("observation")>-1 ) {
620
	                type = SpecimenOrObservationType.Observation;
621
	            }
622
	            if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("fossil")>-1){
623
	                type = SpecimenOrObservationType.Fossil;
624
	            }
625
	            if (state.getDataHolder().getRecordBasis().toLowerCase().indexOf("living")>-1) {
626
	                type = SpecimenOrObservationType.LivingSpecimen;
627
	            }
628
	            if (type == null) {
629
	                logger.info("The basis of record does not seem to be known: " + state.getDataHolder().getRecordBasis());
630
	                type = SpecimenOrObservationType.DerivedUnit;
631
	            }
632
	            // TODO fossils?
633
	        } else {
634
	            logger.info("The basis of record is null");
635
	            type = SpecimenOrObservationType.DerivedUnit;
636
	        }
637
	        DerivedUnitFacade derivedUnitFacade = DerivedUnitFacade.NewInstance(type);
638
	        return derivedUnitFacade;
639
	    }
640

    
641
	    /**
642
	     * Look if the Institution does already exist
643
	     * @param institutionCode: a string with the institutioncode
644
	     * @param config : the configurator
645
	     * @return the Institution (existing or new)
646
	     */
647
	    protected Institution getInstitution(String institutionCode, STATE state) {
648
	        SpecimenImportConfiguratorBase config = state.getConfig();
649
	        Institution institution=null;
650
	        institution = (Institution)state.institutions.get(institutionCode);
651
	        if (institution != null){
652
	            return institution;
653
	        }
654
	        List<Institution> institutions;
655
	        try {
656
	            institutions = getAgentService().searchInstitutionByCode(institutionCode);
657

    
658
	        } catch (Exception e) {
659
	            institutions = new ArrayList<Institution>();
660
	            logger.warn(e);
661
	        }
662
	        if (institutions.size() > 0 && config.isReuseExistingMetaData()) {
663
	            for (Institution institut:institutions){
664
	                try{
665
	                    if (institut.getCode().equalsIgnoreCase(institutionCode)) {
666
	                        institution=institut;
667
	                        break;
668
	                    }
669
	                }catch(Exception e){logger.warn("no institution code in the db");}
670
	            }
671
	        }
672
	        if (logger.isDebugEnabled()){
673
	            if(institution !=null) {
674
	                logger.info("getinstitution " + institution.toString());
675
	            }
676
	        }
677
	        if (institution == null){
678
	            // create institution
679
	            institution = Institution.NewInstance();
680
	            institution.setCode(institutionCode);
681
	            institution.setTitleCache(institutionCode, true);
682
	            UUID uuid = save(institution, state);
683
	        }
684

    
685

    
686
	        state.institutions.put(institutionCode, institution);
687
	        return institution;
688
	    }
689

    
690
	    /**
691
	     * Look if the Collection does already exist
692
	     * @param collectionCode
693
	     * @param collectionCode: a string
694
	     * @param config : the configurator
695
	     * @return the Collection (existing or new)
696
	     */
697
	    protected Collection getCollection(Institution institution, String collectionCode, STATE state) {
698
	        SpecimenImportConfiguratorBase config = state.getConfig();
699
	        Collection collection = null;
700
	        List<Collection> collections;
701
	        collection = (Collection) state.collections.get(collectionCode);
702
	        if (collection != null){
703
	            return collection;
704
	        }
705
	        try {
706
	            collections = getCollectionService().searchByCode(collectionCode);
707
	        } catch (Exception e) {
708
	            collections = new ArrayList<Collection>();
709
	        }
710
	        if (collections.size() > 0 && config.isReuseExistingMetaData()) {
711
	            for (Collection coll:collections){
712
	                if (coll.getCode() != null && coll.getInstitute() != null
713
	                        && coll.getCode().equalsIgnoreCase(collectionCode) && coll.getInstitute().equals(institution)) {
714
	                    collection = coll;
715
	                    break;
716
	                }
717
	            }
718
	        }
719

    
720
	        if(collection == null){
721
	            collection =Collection.NewInstance();
722
	            collection.setCode(collectionCode);
723
	            collection.setInstitute(institution);
724
	            collection.setTitleCache(collectionCode);
725
	            UUID uuid = save(collection, state);
726
	        }
727

    
728

    
729

    
730
	        state.collections.put(collectionCode, collection);
731

    
732
	        return collection;
733
	    }
734

    
735
	    /**
736
	     * @param reference
737
	     * @param citationDetail
738
	     * @return
739
	     */
740
	    //FIXME this method is highly critical, because
741
	    //  * it will have serious performance and memory problems with large databases
742
	    //        (databases may easily have >1 Mio source records)
743
	    //  * it does not make sense to search for existing sources and then clone them
744
	    //    we need to search for existing references instead and use them (if exist)
745
	    //    for our new source.
746
	    protected IdentifiableSource getIdentifiableSource(Reference reference, String citationDetail) {
747

    
748
	      /*  List<IdentifiableSource> issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
749

    
750

    
751
	        if (reference != null){
752
	            try {
753
	                for (OriginalSourceBase<?> osb: issTmp){
754
	                    if (osb.getCitation() != null && osb.getCitation().getTitleCache().equalsIgnoreCase(reference.getTitleCache())){
755
	                        String osbDetail = osb.getCitationMicroReference();
756
	                        if ((StringUtils.isBlank(osbDetail) && StringUtils.isBlank(citationDetail))
757
	                                || (osbDetail != null && osbDetail.equalsIgnoreCase(citationDetail)) ) {
758
//	                            System.out.println("REFERENCE FOUND RETURN EXISTING SOURCE");
759
	                            return (IdentifiableSource) osb.clone();
760
	                        }
761
	                    }
762
	                }
763
	            } catch (CloneNotSupportedException e) {
764
	                throw new RuntimeException(e);
765
	            } catch (Exception e1){
766
	                e1.printStackTrace();
767
	            }
768
	        }
769
	*/
770
	        IdentifiableSource sour = IdentifiableSource.NewInstance(OriginalSourceType.Import,null,null, reference,citationDetail);
771
	        return sour;
772
	    }
773

    
774
	    /**
775
	     * Add the hierarchy for a Taxon(add higher taxa)
776
	     * @param classification
777
	     * @param taxon: a taxon to add as a node
778
	     * @param state: the ABCD import state
779
	     */
780
	    protected void addParentTaxon(Taxon taxon, STATE state, boolean preferredFlag, Classification classification){
781
	        INonViralName  nvname = taxon.getName();
782
	        Rank rank = nvname.getRank();
783
	        Taxon genus =null;
784
	        Taxon subgenus =null;
785
	        Taxon species = null;
786
	        Taxon subspecies = null;
787
	        Taxon parent = null;
788
	        if(rank!=null){
789
	            if (rank.isLower(Rank.GENUS() )){
790
	                String genusOrUninomial = nvname.getGenusOrUninomial();
791
	                TaxonName taxonName = getOrCreateTaxonName(genusOrUninomial, Rank.GENUS(), preferredFlag, state, -1);
792
	                genus = getOrCreateTaxonForName(taxonName, state);
793
	                if (genus == null){
794
	                    logger.debug("The genus should not be null " + taxonName);
795
	                }
796
	                if (preferredFlag) {
797
	                    parent = linkParentChildNode(null, genus, classification, state);
798
	                }
799

    
800
	            }
801
	            if (rank.isLower(Rank.SUBGENUS())){
802
	                String prefix = nvname.getGenusOrUninomial();
803
	                String name = nvname.getInfraGenericEpithet();
804
	                if (name != null){
805
	                    TaxonName taxonName = getOrCreateTaxonName(prefix+" "+name, Rank.SUBGENUS(), preferredFlag, state, -1);
806
	                    subgenus = getOrCreateTaxonForName(taxonName, state);
807
	                    if (preferredFlag) {
808
	                        parent = linkParentChildNode(genus, subgenus, classification, state);
809
	                    }            }
810
	            }
811
	            if (rank.isLower(Rank.SPECIES())){
812
	                if (subgenus!=null){
813
	                    String prefix = nvname.getGenusOrUninomial();
814
	                    String name = nvname.getInfraGenericEpithet();
815
	                    String spe = nvname.getSpecificEpithet();
816
	                    if (spe != null){
817
	                        TaxonName taxonName = getOrCreateTaxonName(prefix+" "+name+" "+spe, Rank.SPECIES(), preferredFlag, state, -1);
818
	                        species = getOrCreateTaxonForName(taxonName, state);
819
	                        if (preferredFlag) {
820
	                            parent = linkParentChildNode(subgenus, species, classification, state);
821
	                        }
822
	                    }
823
	                }
824
	                else{
825
	                    String prefix = nvname.getGenusOrUninomial();
826
	                    String name = nvname.getSpecificEpithet();
827
	                    if (name != null){
828
	                        TaxonName taxonName = getOrCreateTaxonName(prefix+" "+name, Rank.SPECIES(), preferredFlag, state, -1);
829
	                        species = getOrCreateTaxonForName(taxonName, state);
830
	                        if (preferredFlag) {
831
	                            parent = linkParentChildNode(genus, species, classification, state);
832
	                        }
833
	                    }
834
	                }
835
	            }
836
	            if (rank.isLower(Rank.INFRASPECIES())){
837
	                TaxonName taxonName = getOrCreateTaxonName(nvname.getFullTitleCache(), Rank.SUBSPECIES(), preferredFlag, state, -1);
838
	                subspecies = getOrCreateTaxonForName(taxonName, state);
839
	                if (preferredFlag) {
840
	                    parent = linkParentChildNode(species, subspecies, classification, state);
841
	                }
842
	            }
843
	        }
844
	        if (preferredFlag && parent!=taxon ) {
845
	            linkParentChildNode(parent, taxon, classification, state);
846
	        }
847
	    }
848

    
849
	    /**
850
	     * Link a parent to a child and save it in the current classification
851
	     * @param parent: the higher Taxon
852
	     * @param child : the lower (or current) Taxon
853
	     * return the Taxon from the new created Node
854
	     * @param classification
855
	     * @param state
856
	     */
857
	    protected Taxon linkParentChildNode(Taxon parent, Taxon child, Classification classification, STATE state) {
858
	        TaxonNode node =null;
859
	        if (parent != null) {
860
	            parent = (Taxon) getTaxonService().find(parent.getUuid());
861
	            child = (Taxon) getTaxonService().find(child.getUuid());
862
	            //here we do not have to check if the taxon nodes already exists
863
	            //this is done by classification.addParentChild()
864
	            //do not add child node if it already exists
865
	            if(hasTaxonNodeInClassification(child, classification)){
866
	                return child;
867
	            }
868
	            else{
869
	                node = classification.addParentChild(parent, child, state.getRef(), "");
870
	                save(node, state);
871
	            }
872
	        }
873
	        else {
874
	            if (child == null){
875
	                logger.debug("The child should not be null!");
876
	            }
877
	            child = (Taxon) getTaxonService().find(child.getUuid());
878
	            //do not add child node if it already exists
879
	            if(hasTaxonNodeInClassification(child, classification)){
880
	                return child;
881
	            }
882
	            else{
883
	                node = classification.addChildTaxon(child, state.getRef(), null);
884
	                save(node, state);
885
	            }
886
	        }
887
	        if(node!=null){
888
	            state.getReport().addTaxonNode(node);
889
	            return node.getTaxon();
890
	        }
891
	        String message = "Could not create taxon node for " +child;
892
	        state.getReport().addInfoMessage(message);
893
	        logger.warn(message);
894
	        return null;
895
	    }
896

    
897
	    protected Taxon getOrCreateTaxonForName(TaxonName taxonName, STATE state){
898
	        if (taxonName != null){
899
    	        Set<Taxon> acceptedTaxa = taxonName.getTaxa();
900
    	        if(acceptedTaxa.size()>0){
901
    	            Taxon firstAcceptedTaxon = acceptedTaxa.iterator().next();
902
    	            if(acceptedTaxa.size()>1){
903
    	                String message = "More than one accepted taxon was found for taxon name: "
904
    	                        + taxonName.getTitleCache() + "!\n" + firstAcceptedTaxon + "was chosen for "+state.getDerivedUnitBase();
905
    	                state.getReport().addInfoMessage(message);
906
    	                logger.warn(message);
907
    	            }
908
    	            else{
909
    	                return firstAcceptedTaxon;
910
    	            }
911
    	        }
912
    	        else{
913
    	            Set<TaxonBase> taxonAndSynonyms = taxonName.getTaxonBases();
914
    	            for (TaxonBase taxonBase : taxonAndSynonyms) {
915
    	                if(taxonBase.isInstanceOf(Synonym.class)){
916
    	                    Synonym synonym = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
917
    	                    Taxon acceptedTaxonOfSynonym = synonym.getAcceptedTaxon();
918
    	                    if(acceptedTaxonOfSynonym == null){
919
    	                        String message = "No accepted taxon could be found for taxon name: "
920
    	                                + taxonName.getTitleCache()
921
    	                                + "!";
922
    	                        state.getReport().addInfoMessage(message);
923
    	                        logger.warn(message);
924
    	                    }
925
    	                    else{
926
    	                        return acceptedTaxonOfSynonym;
927
    	                    }
928
    	                }
929
    	            }
930
    	        }
931
    	        Taxon taxon = Taxon.NewInstance(taxonName, state.getRef());
932
    	        save(taxon, state);
933
    	        state.getReport().addTaxon(taxon);
934
    	        logger.info("Created new taxon "+ taxon);
935
    	        return taxon;
936
	        }
937
	        return null;
938

    
939
	    }
940

    
941
	    private boolean hasTaxonNodeInClassification(Taxon taxon, Classification classification){
942
	        if(taxon.getTaxonNodes()!=null){
943
	            for (TaxonNode node : taxon.getTaxonNodes()){
944
	                if(node.getClassification().equals(classification)){
945
	                    return true;
946
	                }
947
	            }
948
	        }
949
	        return false;
950
	    }
951

    
952
	    /**
953
	     * HandleIdentifications : get the scientific names present in the ABCD
954
	     * document and store link them with the observation/specimen data
955
	     * @param state: the current ABCD import state
956
	     * @param derivedUnitFacade : the current derivedunitfacade
957
	     */
958
	    protected void handleIdentifications(STATE state, DerivedUnitFacade derivedUnitFacade) {
959
	       SpecimenImportConfiguratorBase config = state.getConfig();
960

    
961

    
962
	        String scientificName = "";
963
	        boolean preferredFlag = false;
964

    
965
	        if (state.getDataHolder().getNomenclatureCode() == ""){
966
	            if (config.getNomenclaturalCode() != null){
967
	                if (config.getNomenclaturalCode() != null){
968
	                    state.getDataHolder().setNomenclatureCode(config.getNomenclaturalCode().toString());
969

    
970
	                }
971
	            }
972
	        }
973

    
974
	        for (int i = 0; i < state.getDataHolder().getIdentificationList().size(); i++) {
975
	            Identification identification = state.getDataHolder().getIdentificationList().get(i);
976
	            scientificName = identification.getScientificName().replaceAll(" et ", " & ");
977

    
978
	            String preferred = identification.getPreferred();
979
	            preferredFlag = false;
980
	            if (preferred != null){
981
    	            if (preferred.equals("1") || preferred.toLowerCase().indexOf("true") != -1 || state.getDataHolder().getIdentificationList().size()==1) {
982
    	                preferredFlag = true;
983
    	            }
984

    
985
	            }
986
	            if (identification.getCode() != null){
987
    	            if (identification.getCode().indexOf(':') != -1) {
988
    	                state.getDataHolder().setNomenclatureCode(identification.getCode().split(COLON)[1]);
989
    	            }
990
    	            else{
991
    	                state.getDataHolder().setNomenclatureCode(identification.getCode());
992
    	            }
993
	            }
994
	            TaxonName taxonName = getOrCreateTaxonName(scientificName, null, preferredFlag, state, i);
995
	            Taxon taxon = getOrCreateTaxonForName(taxonName, state);
996
	            addTaxonNode(taxon, state,preferredFlag);
997
	            linkDeterminationEvent(state, taxon, preferredFlag, derivedUnitFacade, identification.getIdentifier(), identification.getDate());
998
	        }
999
	    }
1000

    
1001
	    /**
1002
	     * @param taxon : a taxon to add as a node
1003
	     * @param state : the ABCD import state
1004
	     */
1005
	    protected void addTaxonNode(Taxon taxon, STATE state, boolean preferredFlag) {
1006
	        SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
1007
	        logger.info("link taxon to a taxonNode "+taxon.getTitleCache());
1008
	        //only add nodes if not already existing in current classification or default classification
1009

    
1010
	        //check if node exists in current classification
1011
	        //NOTE: we cannot use hasTaxonNodeInClassification() here because we are first creating it here
1012
	        if (!existsInClassification(taxon,state.getClassification(), state)){
1013
	            if(config.isMoveNewTaxaToDefaultClassification()){
1014
	                //check if node exists in default classification
1015
	                if (!existsInClassification(taxon, state.getDefaultClassification(true), state)){
1016
	                    addParentTaxon(taxon, state, preferredFlag, state.getDefaultClassification(true));
1017
	                }
1018
	            }else{
1019
	                //add non-existing taxon to current classification
1020
	                addParentTaxon(taxon, state, preferredFlag, state.getClassification());
1021
	            }
1022

    
1023
	        }
1024
	    }
1025

    
1026

    
1027
	    private boolean existsInClassification(Taxon taxon, Classification classification, STATE state){
1028
	        boolean exist = false;
1029
	        ICdmRepository cdmAppController = state.getConfig().getCdmAppController();
1030
            if(cdmAppController==null){
1031
                cdmAppController = this;
1032
            }
1033
            if (classification != null){
1034
                if (!taxon.getTaxonNodes().isEmpty()){
1035
                    for (TaxonNode node:taxon.getTaxonNodes()){
1036
                        if (node.getClassification().equals(classification)){
1037
                            return true;
1038
                        }
1039
                    }
1040
                }
1041
// we do not need this because we already searched for taxa in db in the previous steps
1042
//    	        List<UuidAndTitleCache<TaxonNode>> uuidAndTitleCacheOfAllTaxa = cdmAppController.getClassificationService().getTaxonNodeUuidAndTitleCacheOfAcceptedTaxaByClassification(classification.getUuid());
1043
//    	        if (uuidAndTitleCacheOfAllTaxa != null){
1044
//        	        for (UuidAndTitleCache p : uuidAndTitleCacheOfAllTaxa){
1045
//        	            try{
1046
//        	                if(p.getTitleCache().equals(taxon.getTitleCache())) {
1047
//        	                    exist = true;
1048
//        	                }
1049
//        	            }
1050
//        	            catch(Exception e){
1051
//        	                logger.warn("TaxonNode doesn't seem to have a taxon");
1052
//        	            }
1053
//        	        }
1054
//    	        }
1055
            }
1056
	        return exist;
1057
	    }
1058

    
1059
	    /**
1060
	     * join DeterminationEvent to the Taxon Object
1061
	     * @param state : the ABCD import state
1062
	     * @param taxon: the current Taxon
1063
	     * @param preferredFlag :if the current name is preferred
1064
	     * @param derivedFacade : the derived Unit Facade
1065
	     */
1066
	    @SuppressWarnings("rawtypes")
1067
        protected void linkDeterminationEvent(STATE state, Taxon taxon, boolean preferredFlag,  DerivedUnitFacade derivedFacade, String identifierStr, String dateStr) {
1068
	        SpecimenImportConfiguratorBase config = state.getConfig();
1069
	        if (logger.isDebugEnabled()){
1070
	            logger.info("start linkdetermination with taxon:" + taxon.getUuid()+", "+taxon);
1071
	        }
1072

    
1073
	        DeterminationEvent determinationEvent = DeterminationEvent.NewInstance();
1074
	        determinationEvent.setTaxonName(taxon.getName());
1075
	        determinationEvent.setPreferredFlag(preferredFlag);
1076

    
1077

    
1078
	        determinationEvent.setIdentifiedUnit(state.getDerivedUnitBase());
1079
	        if (state.getPersonStore().get(identifierStr) != null){
1080
	            determinationEvent.setActor((AgentBase)state.getPersonStore().get(identifierStr));
1081
	        } else if (identifierStr != null){
1082
	            Person identifier = Person.NewTitledInstance(identifierStr);
1083
	            determinationEvent.setActor(identifier);
1084
	        }
1085
	        if (dateStr != null){
1086
	            determinationEvent.setTimeperiod(TimePeriodParser.parseString(dateStr));
1087
	        }
1088
	        state.getDerivedUnitBase().addDetermination(determinationEvent);
1089

    
1090
	        if (logger.isDebugEnabled()){
1091
	            logger.debug("NB TYPES INFO: "+ state.getDataHolder().getStatusList().size());
1092
	        }
1093
	        for (SpecimenTypeDesignationStatus specimenTypeDesignationstatus : state.getDataHolder().getStatusList()) {
1094
	            if (specimenTypeDesignationstatus != null) {
1095
	                if (logger.isDebugEnabled()){
1096
	                    logger.debug("specimenTypeDesignationstatus :"+ specimenTypeDesignationstatus);
1097
	                }
1098

    
1099
	                ICdmRepository cdmAppController = config.getCdmAppController();
1100
	                if(cdmAppController == null){
1101
	                    cdmAppController = this;
1102
	                }
1103
	                specimenTypeDesignationstatus = (SpecimenTypeDesignationStatus) cdmAppController.getTermService().find(specimenTypeDesignationstatus.getUuid());
1104
	                //Designation
1105
	                TaxonName name = taxon.getName();
1106
	                SpecimenTypeDesignation designation = SpecimenTypeDesignation.NewInstance();
1107

    
1108
	                designation.setTypeStatus(specimenTypeDesignationstatus);
1109
	                designation.setTypeSpecimen(state.getDerivedUnitBase());
1110
	                name.addTypeDesignation(designation, true);
1111
	            }
1112
	        }
1113

    
1114
	        for (String[] fullReference : state.getDataHolder().getReferenceList()) {
1115

    
1116

    
1117
	            String strReference=fullReference[0];
1118
	            String citationDetail = fullReference[1];
1119
	            String citationURL = fullReference[2];
1120
	            List<Reference> references = getReferenceService().listByTitle(Reference.class, "strReference", MatchMode.EXACT, null, null, null, null, null);
1121

    
1122
	            if (!references.isEmpty()){
1123
	                Reference reference = null;
1124
	                for (Reference refe: references) {
1125
	                    if (refe.getTitleCache().equalsIgnoreCase(strReference)) {
1126
	                        reference =refe;
1127
	                        break;
1128
	                    }
1129
	                }
1130
	                if (reference ==null){
1131
	                    reference = ReferenceFactory.newGeneric();
1132
	                    reference.setTitleCache(strReference, true);
1133
	                    save(reference, state);
1134
	                }
1135
	                determinationEvent.addReference(reference);
1136
	            }
1137
	        }
1138
	        save(state.getDerivedUnitBase(), state);
1139

    
1140
	        if (config.isAddIndividualsAssociationsSuchAsSpecimenAndObservations() && preferredFlag) {
1141
	            //do not add IndividualsAssociation to non-preferred taxa
1142
	            if (logger.isDebugEnabled()){
1143
	                logger.debug("isDoCreateIndividualsAssociations");
1144
	            }
1145

    
1146
	            makeIndividualsAssociation(state, taxon, determinationEvent);
1147

    
1148
	            save(state.getDerivedUnitBase(), state);
1149
	        }
1150
	    }
1151

    
1152
	    /**
1153
	     * create and link each association (specimen, observation..) to the accepted taxon
1154
	     * @param state : the ABCD import state
1155
	     * @param taxon: the current Taxon
1156
	     * @param determinationEvent:the determinationevent
1157
	     */
1158
	    protected void makeIndividualsAssociation(STATE state, Taxon taxon, DeterminationEvent determinationEvent) {
1159
	        SpecimenImportConfiguratorBase<?,?,?> config = state.getConfig();
1160
	        SpecimenUserInteraction sui = config.getSpecimenUserInteraction();
1161

    
1162
	        if (logger.isDebugEnabled()){
1163
	            logger.info("MAKE INDIVIDUALS ASSOCIATION");
1164
	        }
1165

    
1166
	        TaxonDescription taxonDescription = null;
1167
	        Set<TaxonDescription> descriptions= taxon.getDescriptions();
1168
	       if (!descriptions.isEmpty()){ taxonDescription = descriptions.iterator().next();}
1169

    
1170
//	            for (TaxonDescription description : descriptions){
1171
//	                Set<IdentifiableSource> sources =  new HashSet<>();
1172
//	                sources.addAll(description.getTaxon().getSources());
1173
//	                sources.addAll(description.getSources());
1174
//	                for (IdentifiableSource source:sources){
1175
//	                    if(state.getRef().equals(source.getCitation())) {
1176
//	                        taxonDescription = description;
1177
//	                    }
1178
//	                }
1179
//	            }
1180
	      //  }
1181
	        if (taxonDescription == null){
1182
	            taxonDescription = TaxonDescription.NewInstance(taxon, false);
1183
	            if(sourceNotLinkedToElement(taxonDescription,state.getRef(),null)) {
1184
	                taxonDescription.addSource(OriginalSourceType.Import, null, null, state.getRef(), null);
1185
	            }
1186
	            state.setDescriptionGroup(taxonDescription);
1187
	            taxon.addDescription(taxonDescription);
1188
	        }
1189

    
1190
	        //PREPARE REFERENCE QUESTIONS
1191

    
1192
	        Map<String,OriginalSourceBase<?>> sourceMap = new HashMap<String, OriginalSourceBase<?>>();
1193

    
1194
	        List<IdentifiableSource> issTmp = new ArrayList<>();//getCommonService().list(IdentifiableSource.class, null, null, null, null);
1195
	        List<DescriptionElementSource> issTmp2 = new ArrayList<>();//getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1196

    
1197
	        Set<OriginalSourceBase> osbSet = new HashSet<OriginalSourceBase>();
1198
	        if(issTmp2!=null) {
1199
	            osbSet.addAll(issTmp2);
1200
	        }
1201
	        if(issTmp!=null) {
1202
	            osbSet.addAll(issTmp);
1203
	        }
1204

    
1205

    
1206
	        addToSourceMap(sourceMap, osbSet);
1207

    
1208
//	        if (((Abcd206ImportConfigurator) state.getConfig()).isInteractWithUser()){
1209
//	            List<OriginalSourceBase<?>> res = null;
1210
//	            if(!state.isDescriptionSourcesSet()){
1211
//	                res = sui.askForSource(sourceMap, "the description group ("+taxon+")",
1212
//	                        "The current reference is "+state.getRef().getTitleCache(),getReferenceService(), state.getDataHolder().docSources);
1213
//	                state.setDescriptionRefs(res);
1214
//	                state.setDescriptionSourcesSet(true);
1215
//	            }
1216
//	            else{
1217
//	                res=state.getDescriptionRefs();
1218
//	            }
1219
//	            if(res !=null) {
1220
//	                for (OriginalSourceBase<?> sour:res){
1221
//	                    if(sour.isInstanceOf(IdentifiableSource.class)){
1222
//	                        try {
1223
//	                            if(sourceNotLinkedToElement(taxonDescription,sour)) {
1224
//	                                taxonDescription.addSource((IdentifiableSource)sour.clone());
1225
//	                            }
1226
//	                        } catch (CloneNotSupportedException e) {
1227
//	                            logger.warn("no cloning?");
1228
//	                        }
1229
//	                    }else{
1230
//	                        if(sourceNotLinkedToElement(taxonDescription,sour)) {
1231
//	                            taxonDescription.addSource(OriginalSourceType.Import,null, null, sour.getCitation(),sour.getCitationMicroReference());
1232
//	                        }
1233
//	                    }
1234
//	                }
1235
//	            }
1236
//	        }
1237
//	        else {
1238
	            if(sourceNotLinkedToElement(taxonDescription,state.getRef(),null)) {
1239
	                taxonDescription.addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1240
	            }
1241
//	        }
1242
	        state.setDescriptionGroup(taxonDescription);
1243

    
1244
	        IndividualsAssociation indAssociation = IndividualsAssociation.NewInstance();
1245
	        Feature feature = makeFeature(state.getDerivedUnitBase());
1246
	        indAssociation.setAssociatedSpecimenOrObservation(state.getDerivedUnitBase());
1247
	        indAssociation.setFeature(feature);
1248
//
1249
//	        if (((Abcd206ImportConfigurator) state.getConfig()).isInteractWithUser()){
1250
//	            sourceMap = new HashMap<String, OriginalSourceBase<?>>();
1251
//
1252
//	            issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
1253
//	            issTmp2 = getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1254
//
1255
//	            osbSet = new HashSet<OriginalSourceBase>();
1256
//	            if(issTmp2!=null) {
1257
//	                osbSet.addAll(issTmp2);
1258
//	            }
1259
//	            if(issTmp!=null) {
1260
//	                osbSet.addAll(issTmp);
1261
//	            }
1262
//
1263
//
1264
//	            addToSourceMap(sourceMap, osbSet);
1265
//
1266
//	            List<OriginalSourceBase<?>> sources =null;
1267
//	            if(!state.isAssociationSourcesSet()) {
1268
//	                sources = sui.askForSource(sourceMap,  "descriptive element (association) ",taxon.toString(),
1269
//	                        getReferenceService(),state.getDataHolder().getDocSources());
1270
//	                state.setAssociationRefs(sources);
1271
//	                state.setAssociationSourcesSet(true);
1272
//	            }
1273
//	            else{
1274
//	                sources=state.getAssociationRefs();
1275
//	            }
1276
//	            if(sources !=null) {
1277
//	                for (OriginalSourceBase<?> source: sources) {
1278
//	                    if(source !=null) {
1279
//	                        if(source.isInstanceOf(DescriptionElementSource.class)){
1280
//	                            try {
1281
//	                                if(sourceNotLinkedToElement(indAssociation,source)) {
1282
//	                                    indAssociation.addSource((DescriptionElementSource)source.clone());
1283
//	                                }
1284
//	                            } catch (CloneNotSupportedException e) {
1285
//	                                logger.warn("clone forbidden?");
1286
//	                            }
1287
//	                        }else{
1288
//	                            if(sourceNotLinkedToElement(indAssociation,source)) {
1289
//	                                indAssociation.addSource(OriginalSourceType.Import,null, null, source.getCitation(),source.getCitationMicroReference());
1290
//	                            }
1291
//	                            try {
1292
//	                                if(sourceNotLinkedToElement(state.getDerivedUnitBase(), source)) {
1293
//	                                    state.getDerivedUnitBase().addSource((IdentifiableSource) source.clone());
1294
//	                                }
1295
//	                            } catch (CloneNotSupportedException e) {
1296
//	                                // TODO Auto-generated catch block
1297
//	                                e.printStackTrace();
1298
//	                            }
1299
//	                        }
1300
//
1301
//	                    }
1302
//	                }
1303
//	            }
1304
//	        }else {
1305
	            if(sourceNotLinkedToElement(indAssociation,state.getRef(),null)) {
1306
	                indAssociation.addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1307
	            }
1308
	            if(sourceNotLinkedToElement(state.getDerivedUnitBase(), state.getRef(),null)) {
1309
	                state.getDerivedUnitBase().addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1310
	            }
1311
	            for (Reference citation : determinationEvent.getReferences()) {
1312
	                if(sourceNotLinkedToElement(indAssociation,citation,null))
1313
	                {
1314
	                    indAssociation.addSource(DescriptionElementSource.NewInstance(OriginalSourceType.Import, null, null, citation, null));
1315
	                }
1316
	                if(sourceNotLinkedToElement(state.getDerivedUnitBase(), state.getRef(),null)) {
1317
	                    state.getDerivedUnitBase().addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1318
	                }
1319
	            }
1320
	   //     }
1321

    
1322
	        taxonDescription.addElement(indAssociation);
1323

    
1324
	        save(taxonDescription, state);
1325
	        save(taxon, state);
1326
	        state.getReport().addDerivate(state.getDerivedUnitBase(), config);
1327
	        state.getReport().addIndividualAssociation(taxon, state.getDataHolder().getUnitID(), state.getDerivedUnitBase());
1328
	    }
1329

    
1330
	    /**
1331
	     * @param derivedUnitBase2
1332
	     * @param ref2
1333
	     * @param object
1334
	     * @return
1335
	     */
1336
	    private boolean sourceNotLinkedToElement(DerivedUnit derivedUnitBase2, Reference b, String d) {
1337
	        Set<IdentifiableSource> linkedSources = derivedUnitBase2.getSources();
1338
	        for (IdentifiableSource is:linkedSources){
1339
	            Reference a = is.getCitation();
1340
	            String c = is.getCitationMicroReference();
1341

    
1342
	            boolean refMatch=false;
1343
	            boolean microMatch=false;
1344

    
1345
	            try{
1346
	                if (a==null && b==null) {
1347
	                    refMatch=true;
1348
	                }
1349
	                if (a!=null && b!=null) {
1350
	                    if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1351
	                        refMatch=true;
1352
	                    }
1353
	                }
1354
	            }catch(Exception e){}
1355

    
1356

    
1357
	            try{
1358
	                if (c==null && d==null) {
1359
	                    microMatch=true;
1360
	                }
1361
	                if(c!=null && d!=null) {
1362
	                    if(c.equalsIgnoreCase(d)) {
1363
	                        microMatch=true;
1364
	                    }
1365
	                }
1366
	            }
1367
	            catch(Exception e){}
1368

    
1369
	            if (microMatch && refMatch) {
1370
	                return false;
1371
	            }
1372

    
1373

    
1374
	        }
1375
	        return true;
1376
	    }
1377

    
1378
	    private <T extends OriginalSourceBase<?>> boolean  sourceNotLinkedToElement(ISourceable<T> sourcable, Reference reference, String microReference) {
1379
	        Set<T> linkedSources = sourcable.getSources();
1380
	        for (T is:linkedSources){
1381
	            Reference unitReference = is.getCitation();
1382
	            String unitMicroReference = is.getCitationMicroReference();
1383

    
1384
	            boolean refMatch=false;
1385
	            boolean microMatch=false;
1386

    
1387
	            try{
1388
	                if (unitReference==null && reference==null) {
1389
	                    refMatch=true;
1390
	                }
1391
	                if (unitReference!=null && reference!=null) {
1392
	                    if (unitReference.getTitleCache().equalsIgnoreCase(reference.getTitleCache())) {
1393
	                        refMatch=true;
1394
	                    }
1395
	                }
1396
	            }catch(Exception e){}
1397

    
1398
	            try{
1399
	                if (unitMicroReference==null && microReference==null) {
1400
	                    microMatch=true;
1401
	                }
1402
	                if(unitMicroReference!=null && microReference!=null) {
1403
	                    if(unitMicroReference.equalsIgnoreCase(microReference)) {
1404
	                        microMatch=true;
1405
	                    }
1406
	                }
1407
	            }
1408
	            catch(Exception e){}
1409

    
1410
	            if (microMatch && refMatch) {
1411
	                return false;
1412
	            }
1413
	        }
1414
	        return true;
1415
	    }
1416

    
1417
	    /**
1418
	     * look for the Feature object (FieldObs, Specimen,...)
1419
	     * @param unit : a specimen or obersvation base
1420
	     * @return the corresponding Feature
1421
	     */
1422
	    private Feature makeFeature(SpecimenOrObservationBase<?> unit) {
1423
	        SpecimenOrObservationType type = unit.getRecordBasis();
1424

    
1425

    
1426

    
1427
	        if (type.isFeatureObservation()){
1428
	            return Feature.OBSERVATION();
1429
	        }else if (type.isFeatureSpecimen()){
1430
	            return Feature.SPECIMEN();
1431
	        }else if (type == SpecimenOrObservationType.DerivedUnit){
1432
	            return Feature.OBSERVATION();
1433
	            //            return getFeature("Specimen or observation");
1434
	        }else{
1435
	            String message = "Unhandled record basis '%s' for defining individuals association feature type. Use default.";
1436
	            logger.warn(String.format(message, type.getMessage()));
1437
	            return Feature.OBSERVATION();
1438
	            //            return getFeature("Specimen or observation");
1439

    
1440
	        }
1441
	    }
1442

    
1443

    
1444
	    /**
1445
	     * @param sourceMap
1446
	     * @param osbSet
1447
	     */
1448
	    protected void addToSourceMap(Map<String, OriginalSourceBase<?>> sourceMap, Set<OriginalSourceBase> osbSet) {
1449
	        for( OriginalSourceBase<?> osb:osbSet) {
1450
	            if(osb.getCitation()!=null && osb.getCitationMicroReference() !=null  && !osb.getCitationMicroReference().isEmpty()) {
1451
	                try{
1452
	                    sourceMap.put(osb.getCitation().getTitleCache()+ "---"+osb.getCitationMicroReference(),osb);
1453
	                }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1454
	            } else if(osb.getCitation()!=null){
1455
	                try{
1456
	                    sourceMap.put(osb.getCitation().getTitleCache(),osb);
1457
	                }catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
1458
	            }
1459
	        }
1460
	    }
1461

    
1462

    
1463
}
(2-2/7)