/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/abcd206/in/Abcd206Import.java - Annotate - EDIT - EDIT Project Management

| Branch: | Tag: | Revision:

cdmlib/cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/specimen/abcd206/in/Abcd206Import.java @ c95e3daf

1	636583bf	Andreas Müller	/**
2	eb82b680	p.kelbert	* Copyright (C) 2007 EDIT
3	1d3f5c87	p.kelbert	* European Distributed Institute of Taxonomy
4	eb82b680	p.kelbert	* http://www.e-taxonomy.eu
5	1d3f5c87	p.kelbert	*
6	eb82b680	p.kelbert	* The contents of this file are subject to the Mozilla Public License Version 1.1
7			* See LICENSE.TXT at the top of this package for the full license terms.
8			*/
9	636583bf	Andreas Müller
10			package eu.etaxonomy.cdm.io.specimen.abcd206.in;
11
12	eb82b680	p.kelbert	import java.net.MalformedURLException;
13	636583bf	Andreas Müller	import java.util.ArrayList;
14			import java.util.HashMap;
15	be709878	p.kelbert	import java.util.HashSet;
16	636583bf	Andreas Müller	import java.util.List;
17	be709878	p.kelbert	import java.util.Map;
18	636583bf	Andreas Müller	import java.util.Set;
19	be709878	p.kelbert	import java.util.UUID;
20	636583bf	Andreas Müller
21	eb82b680	p.kelbert	import org.apache.commons.lang.StringUtils;
22	636583bf	Andreas Müller	import org.apache.log4j.Logger;
23			import org.springframework.stereotype.Component;
24			import org.w3c.dom.Element;
25			import org.w3c.dom.NodeList;
26
27	0092a4a5	Cherian Mathew	import eu.etaxonomy.cdm.api.application.ICdmApplicationConfiguration;
28	f0008608	Andreas Müller	import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
29	742aad84	Patric Plitzner	import eu.etaxonomy.cdm.api.service.config.FindOccurrencesConfigurator;
30			import eu.etaxonomy.cdm.api.service.pager.Pager;
31	ce8437b0	Patric Plitzner	import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
32	f38990b2	Andreas Müller	import eu.etaxonomy.cdm.io.specimen.SpecimenImportBase;
33	1d3f5c87	p.kelbert	import eu.etaxonomy.cdm.io.specimen.SpecimenUserInteraction;
34	636583bf	Andreas Müller	import eu.etaxonomy.cdm.io.specimen.UnitsGatheringArea;
35			import eu.etaxonomy.cdm.io.specimen.UnitsGatheringEvent;
36	2bc696d1	Patrick Plitzner	import eu.etaxonomy.cdm.io.specimen.abcd206.in.molecular.AbcdDnaParser;
37	be709878	p.kelbert	import eu.etaxonomy.cdm.model.agent.AgentBase;
38	636583bf	Andreas Müller	import eu.etaxonomy.cdm.model.agent.Institution;
39	be709878	p.kelbert	import eu.etaxonomy.cdm.model.agent.Person;
40	636583bf	Andreas Müller	import eu.etaxonomy.cdm.model.agent.Team;
41	be709878	p.kelbert	import eu.etaxonomy.cdm.model.common.CdmBase;
42	4134e828	Patric Plitzner	import eu.etaxonomy.cdm.model.common.DefinedTerm;
43	1d3f5c87	p.kelbert	import eu.etaxonomy.cdm.model.common.DefinedTermBase;
44			import eu.etaxonomy.cdm.model.common.IdentifiableSource;
45	eb82b680	p.kelbert	import eu.etaxonomy.cdm.model.common.Language;
46	f96df645	Andreas Müller	import eu.etaxonomy.cdm.model.common.LanguageString;
47	1de96f38	p.kelbert	import eu.etaxonomy.cdm.model.common.OriginalSourceBase;
48	fad74844	Andreas Müller	import eu.etaxonomy.cdm.model.common.OriginalSourceType;
49	f96df645	Andreas Müller	import eu.etaxonomy.cdm.model.description.DescriptionBase;
50	289d49ad	Andreas Müller	import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
51	3180d917	Andreas Kohlbecker	import eu.etaxonomy.cdm.model.description.Feature;
52			import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
53			import eu.etaxonomy.cdm.model.description.TaxonDescription;
54	636583bf	Andreas Müller	import eu.etaxonomy.cdm.model.location.NamedArea;
55			import eu.etaxonomy.cdm.model.media.Media;
56	89b754ae	Patric Plitzner	import eu.etaxonomy.cdm.model.molecular.DnaSample;
57	636583bf	Andreas Müller	import eu.etaxonomy.cdm.model.name.BacterialName;
58			import eu.etaxonomy.cdm.model.name.BotanicalName;
59			import eu.etaxonomy.cdm.model.name.CultivarPlantName;
60			import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
61			import eu.etaxonomy.cdm.model.name.NonViralName;
62			import eu.etaxonomy.cdm.model.name.Rank;
63	eb82b680	p.kelbert	import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
64			import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
65			import eu.etaxonomy.cdm.model.name.TaxonNameBase;
66	636583bf	Andreas Müller	import eu.etaxonomy.cdm.model.name.ZoologicalName;
67			import eu.etaxonomy.cdm.model.occurrence.Collection;
68	266bc333	Patric Plitzner	import eu.etaxonomy.cdm.model.occurrence.DerivationEvent;
69			import eu.etaxonomy.cdm.model.occurrence.DerivationEventType;
70	eb82b680	p.kelbert	import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
71	636583bf	Andreas Müller	import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
72	2574e41b	Patrick Plitzner	import eu.etaxonomy.cdm.model.occurrence.FieldUnit;
73	f0008608	Andreas Müller	import eu.etaxonomy.cdm.model.occurrence.GatheringEvent;
74	266bc333	Patric Plitzner	import eu.etaxonomy.cdm.model.occurrence.MediaSpecimen;
75	eb82b680	p.kelbert	import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationBase;
76	483579cd	Andreas Müller	import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationType;
77	1d36aa54	Andreas Müller	import eu.etaxonomy.cdm.model.reference.Reference;
78	636583bf	Andreas Müller	import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
79	eb82b680	p.kelbert	import eu.etaxonomy.cdm.model.taxon.Classification;
80	ce8437b0	Patric Plitzner	import eu.etaxonomy.cdm.model.taxon.Synonym;
81	636583bf	Andreas Müller	import eu.etaxonomy.cdm.model.taxon.Taxon;
82			import eu.etaxonomy.cdm.model.taxon.TaxonBase;
83	eb82b680	p.kelbert	import eu.etaxonomy.cdm.model.taxon.TaxonNode;
84	d26d6619	Andreas M��ller	import eu.etaxonomy.cdm.persistence.dto.UuidAndTitleCache;
85	1d3f5c87	p.kelbert	import eu.etaxonomy.cdm.persistence.query.MatchMode;
86	636583bf	Andreas Müller	import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
87
88			/**
89			* @author p.kelbert
90			* @created 20.10.2008
91			*/
92			@Component
93	fa28b537	Patric Plitzner	public class Abcd206Import extends SpecimenImportBase<Abcd206ImportConfigurator, Abcd206ImportState> {
94	e2e537ff	Patric Plitzner	private static final String DEFAULT_CLASSIFICATION_ABCD = "Default Classification ABCD";
95
96	1d3f5c87	p.kelbert	private static final Logger logger = Logger.getLogger(Abcd206Import.class);
97
98	1de96f38	p.kelbert	private final boolean DEBUG = true;
99	1d3f5c87	p.kelbert
100			private static final String COLON = ":";
101			private static String prefix = "";
102
103	ce8437b0	Patric Plitzner	private Abcd206ImportReport report;
104
105	1d3f5c87	p.kelbert	public Abcd206Import() {
106			super();
107			}
108
109			@Override
110			protected boolean doCheck(Abcd206ImportState state) {
111			logger.warn("Checking not yet implemented for " + this.getClass().getSimpleName());
112			return true;
113			}
114
115
116			@Override
117			@SuppressWarnings("rawtypes")
118			public void doInvoke(Abcd206ImportState state) {
119	ce8437b0	Patric Plitzner	report = new Abcd206ImportReport();
120	631a9c22	Patrick Plitzner	ICdmApplicationConfiguration cdmAppController = state.getCdmRepository()!=null?state.getCdmRepository():this;
121	ee0d5259	Patric Plitzner
122	1d3f5c87	p.kelbert	state.setTx(startTransaction());
123			logger.info("INVOKE Specimen Import from ABCD2.06 XML ");
124
125			SpecimenUserInteraction sui = state.getConfig().getSpecimenUserInteraction();
126
127			List<Reference> references = getReferenceService().list(Reference.class, null, null, null, null);
128
129			if (state.getConfig().isInteractWithUser()){
130			Map<String,Reference> refMap = new HashMap<String, Reference>();
131	ca8c03d8	Patric Plitzner	for (Reference reference : references) {
132			if (! StringUtils.isBlank(reference.getTitleCache())) {
133			refMap.put(reference.getTitleCache(),reference);
134	1d3f5c87	p.kelbert	}
135			}
136	64260466	Patrick Plitzner	state.setRef(sui.askForReference(refMap));
137	1d3f5c87	p.kelbert
138	64260466	Patrick Plitzner	if (state.getRef() == null){
139	1d3f5c87	p.kelbert	String cla = sui.createNewReference();
140			if (refMap.get(cla)!= null) {
141	64260466	Patrick Plitzner	state.setRef(refMap.get(cla));
142	1d3f5c87	p.kelbert	} else {
143	64260466	Patrick Plitzner	state.setRef(ReferenceFactory.newGeneric());
144			state.getRef().setTitle(cla);
145	1d3f5c87	p.kelbert	}
146			}
147			else{
148	64260466	Patrick Plitzner	state.setRef(getReferenceService().find(state.getRef().getUuid()));
149	1d3f5c87	p.kelbert	}
150			}else{
151	64260466	Patrick Plitzner	if (state.getRef()==null){
152	1d3f5c87	p.kelbert	String name = NB(state.getConfig().getSourceReferenceTitle());
153	ca8c03d8	Patric Plitzner	for (Reference reference : references) {
154			if (! StringUtils.isBlank(reference.getTitleCache())) {
155			if (reference.getTitleCache().equalsIgnoreCase(name)) {
156	64260466	Patrick Plitzner	state.setRef(reference);
157	1d3f5c87	p.kelbert	}
158			}
159			}
160	64260466	Patrick Plitzner	if (state.getRef() == null){
161			state.setRef(ReferenceFactory.newGeneric());
162			state.getRef().setTitle("ABCD classic");
163	1d3f5c87	p.kelbert	}
164			}
165			}
166	64260466	Patrick Plitzner	save(state.getRef(), state);
167			state.getConfig().setSourceReference(state.getRef());
168	1d3f5c87	p.kelbert
169	1a20a9db	Patric Plitzner	if(state.getConfig().getClassificationUuid()!=null){
170			//load classification from config if it exists
171	64260466	Patrick Plitzner	state.setClassification(getClassificationService().load(state.getConfig().getClassificationUuid()));
172	1a20a9db	Patric Plitzner	}
173	64260466	Patrick Plitzner	if(state.getClassification()==null){//no existing classification was set in config
174	1a20a9db	Patric Plitzner	List<Classification> classificationList = getClassificationService().list(Classification.class, null, null, null, null);
175			//get classification via user interaction
176			if (state.getConfig().isUseClassification() && state.getConfig().isInteractWithUser()){
177			Map<String,Classification> classMap = new HashMap<String, Classification>();
178			for (Classification tree : classificationList) {
179			if (! StringUtils.isBlank(tree.getTitleCache())) {
180			classMap.put(tree.getTitleCache(),tree);
181			}
182	1d3f5c87	p.kelbert	}
183	64260466	Patrick Plitzner	state.setClassification(sui.askForClassification(classMap));
184			if (state.getClassification() == null){
185	1a20a9db	Patric Plitzner	String cla = sui.createNewClassification();
186			if (classMap.get(cla)!= null) {
187	64260466	Patrick Plitzner	state.setClassification(classMap.get(cla));
188	1a20a9db	Patric Plitzner	} else {
189	64260466	Patrick Plitzner	state.setClassification(Classification.NewInstance(cla, state.getRef(), Language.DEFAULT()));
190	1a20a9db	Patric Plitzner	}
191	1d3f5c87	p.kelbert	}
192	64260466	Patrick Plitzner	save(state.getClassification(), state);
193	1d3f5c87	p.kelbert	}
194	1a20a9db	Patric Plitzner	// create default classification
195	64260466	Patrick Plitzner	if (state.getClassification() == null) {
196	1d3f5c87	p.kelbert	String name = NB(state.getConfig().getClassificationName());
197			for (Classification classif : classificationList){
198	0200c616	Patric Plitzner	if (classif.getTitleCache() != null && classif.getCitation() != null
199	64260466	Patrick Plitzner	&& classif.getTitleCache().equalsIgnoreCase(name) && classif.getCitation().equals(state.getRef())) {
200			state.setClassification(classif);
201	1d3f5c87	p.kelbert	}
202			}
203	64260466	Patrick Plitzner	if (state.getClassification() == null){
204			state.setClassification(Classification.NewInstance(name, state.getRef(), Language.DEFAULT()));
205	a8931728	Patric Plitzner	//we do not need a default classification when creating an empty new one
206	64260466	Patrick Plitzner	state.setDefaultClassification(state.getClassification());
207	1d3f5c87	p.kelbert	}
208	64260466	Patrick Plitzner	save(state.getClassification(), state);
209	1d3f5c87	p.kelbert	}
210			}
211
212	c95e3daf	Patrick Plitzner	NodeList unitsList = AbcdParseUtility.getUnitsNodeList(state);
213			prefix = AbcdParseUtility.getPrefix(state);
214	1d3f5c87	p.kelbert
215			if (unitsList != null) {
216			String message = "nb units to insert: " + unitsList.getLength();
217			logger.info(message);
218			updateProgress(state, message);
219	e2e537ff	Patric Plitzner	state.getConfig().getProgressMonitor().beginTask("Importing ABCD file", unitsList.getLength());
220	1d3f5c87	p.kelbert
221	64260466	Patrick Plitzner	state.setDataHolder(new Abcd206DataHolder());
222			state.getDataHolder().reset();
223	1d3f5c87	p.kelbert
224	64260466	Patrick Plitzner	Abcd206XMLFieldGetter abcdFieldGetter = new Abcd206XMLFieldGetter(state.getDataHolder(), prefix);
225	1d3f5c87	p.kelbert
226			prepareCollectors(state, unitsList, abcdFieldGetter);
227
228	64260466	Patrick Plitzner	state.setAssociationRefs(new ArrayList<OriginalSourceBase<?>>());
229			state.setDescriptionRefs(new ArrayList<OriginalSourceBase<?>>());
230			state.setDerivedUnitSources(new ArrayList<OriginalSourceBase<?>>());
231	6490c596	p.kelbert
232	1d3f5c87	p.kelbert	for (int i = 0; i < unitsList.getLength(); i++) {
233	ccd80f9f	Patric Plitzner	if(state.getConfig().getProgressMonitor().isCanceled()){
234			break;
235			}
236	89b754ae	Patric Plitzner	Element item = (Element) unitsList.item(i);
237			this.setUnitPropertiesXML( item, abcdFieldGetter, state);
238	64260466	Patrick Plitzner	updateProgress(state, "Importing data for unit "+state.getDataHolder().unitID+" ("+i+"/"+unitsList.getLength()+")");
239	631a9c22	Patrick Plitzner
240	2574e41b	Patrick Plitzner	// //check if unit is associated to a specimen
241			// SpecimenOrObservationBase<?> existingAssociatedSpecimen = null;
242			// if(state.getDataHolder().associatedUnitIds.size()==1){
243			// String unitId = state.getDataHolder().associatedUnitIds.iterator().next();
244			// existingAssociatedSpecimen = findExistingSpecimen(unitId, state);
245			// }
246			// else{
247			// logger.warn("More than one associated unit found this unit!");
248			// }
249			// if(existingAssociatedSpecimen!=null && existingAssociatedSpecimen.isInstanceOf(DerivedUnit.class)){
250			// state.setDerivedUnitBase(HibernateProxyHelper.deproxy(existingAssociatedSpecimen, DerivedUnit.class));
251			// DerivedUnitFacade facade;
252			// try {
253			// facade = DerivedUnitFacade.NewInstance(state.getDerivedUnitBase());
254			// state.setFieldUnit(facade.getFieldUnit(true));
255			// } catch (DerivedUnitFacadeNotSupportedException e) {
256			// e.printStackTrace();
257			// }
258			// }
259
260	2bc696d1	Patrick Plitzner	// import DNA unit
261	64260466	Patrick Plitzner	if(state.getDataHolder().kindOfUnit!=null && state.getDataHolder().kindOfUnit.equalsIgnoreCase("dna")){
262	631a9c22	Patrick Plitzner	AbcdDnaParser dnaParser = new AbcdDnaParser(prefix, report, cdmAppController);
263	2574e41b	Patrick Plitzner	//parent field unit
264			FieldUnit fieldUnit = FieldUnit.NewInstance();
265			state.setFieldUnit(fieldUnit);
266			DnaSample dnaSample = dnaParser.parse(item, state);
267	2bc696d1	Patrick Plitzner	save(dnaSample, state);
268	631a9c22	Patrick Plitzner	//set dna as derived unit to avoid creating an extra specimen for this dna sample (instead just the field unit will be created)
269			state.setDerivedUnitBase(dnaSample);
270	64260466	Patrick Plitzner	report.addDerivate(AbcdImportUtility.getUnitID(state.getDerivedUnitBase(), state.getConfig()), state.getDerivedUnitBase(), state.getDataHolder().unitID, dnaSample);
271	2bc696d1	Patrick Plitzner	}
272	2574e41b	Patrick Plitzner
273			//import default unit + field unit data
274			this.handleSingleUnit(state);
275
276	631a9c22	Patrick Plitzner	//import associated units
277			NodeList unitAssociationsList = item.getElementsByTagName(prefix+"UnitAssociations");
278			if(unitAssociationsList.getLength()==1 && unitAssociationsList.item(0) instanceof Element){
279			UnitAssociationParser unitAssociationParser = new UnitAssociationParser(prefix, report, cdmAppController);
280			unitAssociationParser.parse((Element)unitAssociationsList.item(0), state);
281			}
282	1d3f5c87	p.kelbert
283	2574e41b	Patrick Plitzner	state.reset();
284	1d3f5c87	p.kelbert	}
285	911dddad	Patric Plitzner	if(state.getConfig().isDeduplicateReferences()){
286	5131af90	Patric Plitzner	getReferenceService().deduplicate(Reference.class, null, null);
287	911dddad	Patric Plitzner	}
288			if(state.getConfig().isDeduplicateClassifications()){
289	5131af90	Patric Plitzner	getClassificationService().deduplicate(Classification.class, null, null);
290			}
291	688d8617	Patric Plitzner	}
292	1d3f5c87	p.kelbert	commitTransaction(state.getTx());
293	8f54e7a5	Patric Plitzner	report.printReport(state.getConfig().getReportUri());
294	1d3f5c87	p.kelbert	return;
295			}
296
297			/**
298			* Handle a single unit
299			* @param state
300			*/
301			@SuppressWarnings("rawtypes")
302			private void handleSingleUnit(Abcd206ImportState state) {
303			if (DEBUG) {
304	64260466	Patrick Plitzner	logger.info("handleSingleUnit "+state.getRef());
305	1d3f5c87	p.kelbert	}
306			try {
307	742aad84	Patric Plitzner	ICdmApplicationConfiguration cdmAppController = state.getConfig().getCdmAppController();
308			if(cdmAppController==null){
309			cdmAppController = this;
310			}
311	207d5d50	Patric Plitzner	//check if unit already exists
312	742aad84	Patric Plitzner	DerivedUnitFacade derivedUnitFacade = null;
313			if(state.getConfig().isIgnoreImportOfExistingSpecimens()){
314	64260466	Patrick Plitzner	SpecimenOrObservationBase<?> existingSpecimen = findExistingSpecimen(state.getDataHolder().unitID, state);
315	207d5d50	Patric Plitzner	if(existingSpecimen!=null && existingSpecimen.isInstanceOf(DerivedUnit.class)){
316	64260466	Patrick Plitzner	state.setDerivedUnitBase(HibernateProxyHelper.deproxy(existingSpecimen, DerivedUnit.class));
317	742aad84	Patric Plitzner	return;
318			}
319			// create facade
320	2574e41b	Patrick Plitzner	if(state.getDerivedUnitBase()==null){
321			derivedUnitFacade = getFacade(state);
322			state.setDerivedUnitBase(derivedUnitFacade.innerDerivedUnit());
323			}
324			else{
325			derivedUnitFacade = DerivedUnitFacade.NewInstance(state.getDerivedUnitBase());
326			}
327	742aad84	Patric Plitzner	}
328			// TODO: implement overwrite/merge specimen
329			// else if(state.getConfig().isOverwriteExistingSpecimens()){
330			// Pager<SpecimenOrObservationBase> existingSpecimens = cdmAppController.getOccurrenceService().findByTitle(config);
331			// if(!existingSpecimens.getRecords().isEmpty()){
332			// derivedUnitFacade = DerivedUnitFacade.NewInstance(derivedUnit);
333			// derivedUnitBase = derivedUnitFacade.innerDerivedUnit();
334			// fieldUnit = derivedUnitFacade.getFieldUnit(true);
335			// }
336			// }
337			else{
338			// create facade
339	2574e41b	Patrick Plitzner	if(state.getDerivedUnitBase()==null){
340			derivedUnitFacade = getFacade(state);
341			state.setDerivedUnitBase(derivedUnitFacade.innerDerivedUnit());
342			}
343			else{
344			derivedUnitFacade = DerivedUnitFacade.NewInstance(state.getDerivedUnitBase());
345			}
346	742aad84	Patric Plitzner	}
347	1d3f5c87	p.kelbert
348			/**
349			* GATHERING EVENT
350			*/
351			// gathering event
352	0fd39e45	Patric Plitzner	UnitsGatheringEvent unitsGatheringEvent = new UnitsGatheringEvent(cdmAppController.getTermService(),
353	64260466	Patrick Plitzner	state.getDataHolder().locality, state.getDataHolder().languageIso, state.getDataHolder().longitude, state.getDataHolder().latitude,
354			state.getDataHolder().gatheringElevationText, state.getDataHolder().gatheringElevationMin,
355			state.getDataHolder().gatheringElevationMax, state.getDataHolder().gatheringElevationUnit, state.getDataHolder().gatheringDateText,
356			state.getDataHolder().gatheringAgentList, state.getDataHolder().gatheringTeamList, state.getConfig());
357	1d3f5c87	p.kelbert
358			// country
359			UnitsGatheringArea unitsGatheringArea = new UnitsGatheringArea();
360	f96df645	Andreas Müller	// unitsGatheringArea.setConfig(state.getConfig(),getOccurrenceService(), getTermService());
361	64260466	Patrick Plitzner	unitsGatheringArea.setParams(state.getDataHolder().isocountry, state.getDataHolder().country, state.getConfig(), cdmAppController.getTermService(), cdmAppController.getOccurrenceService());
362	1d3f5c87	p.kelbert
363			DefinedTermBase<?> areaCountry = unitsGatheringArea.getCountry();
364
365			// other areas
366			unitsGatheringArea = new UnitsGatheringArea();
367			// unitsGatheringArea.setConfig(state.getConfig(),getOccurrenceService(),getTermService());
368	64260466	Patrick Plitzner	unitsGatheringArea.setAreas(state.getDataHolder().namedAreaList,state.getConfig(), cdmAppController.getTermService(), cdmAppController.getVocabularyService());
369	1d3f5c87	p.kelbert	ArrayList<DefinedTermBase> nas = unitsGatheringArea.getAreas();
370			for (DefinedTermBase namedArea : nas) {
371			unitsGatheringEvent.addArea(namedArea);
372			}
373
374			// copy gathering event to facade
375			GatheringEvent gatheringEvent = unitsGatheringEvent.getGatheringEvent();
376			derivedUnitFacade.setLocality(gatheringEvent.getLocality());
377			derivedUnitFacade.setExactLocation(gatheringEvent.getExactLocation());
378			derivedUnitFacade.setCollector(gatheringEvent.getCollector());
379			derivedUnitFacade.setCountry((NamedArea)areaCountry);
380	0fd39e45	Patric Plitzner	derivedUnitFacade.setAbsoluteElevationText(gatheringEvent.getAbsoluteElevationText());
381			derivedUnitFacade.setAbsoluteElevation(gatheringEvent.getAbsoluteElevation());
382			derivedUnitFacade.setAbsoluteElevationMax(gatheringEvent.getAbsoluteElevationMax());
383			derivedUnitFacade.setGatheringPeriod(gatheringEvent.getTimeperiod());
384	1de96f38	p.kelbert
385	1d3f5c87	p.kelbert	for(DefinedTermBase<?> area:unitsGatheringArea.getAreas()){
386			derivedUnitFacade.addCollectingArea((NamedArea) area);
387			}
388			// derivedUnitFacade.addCollectingAreas(unitsGatheringArea.getAreas());
389			// TODO exsiccatum
390
391			// add fieldNumber
392	64260466	Patrick Plitzner	derivedUnitFacade.setFieldNumber(NB(state.getDataHolder().fieldNumber));
393	1d3f5c87	p.kelbert
394			// //add Multimedia URLs
395	64260466	Patrick Plitzner	if (state.getDataHolder().multimediaObjects.size() != -1) {
396			for (String multimediaObject : state.getDataHolder().multimediaObjects) {
397	1d3f5c87	p.kelbert	Media media;
398			try {
399	f379b50b	Andreas Müller	media = getImageMedia(multimediaObject, READ_MEDIA_DATA);
400	1d3f5c87	p.kelbert	derivedUnitFacade.addDerivedUnitMedia(media);
401	4134e828	Patric Plitzner	if(state.getConfig().isAddMediaAsMediaSpecimen()){
402			//add media also as specimen scan
403	266bc333	Patric Plitzner	MediaSpecimen mediaSpecimen = MediaSpecimen.NewInstance(SpecimenOrObservationType.Media);
404			mediaSpecimen.setMediaSpecimen(media);
405	4134e828	Patric Plitzner	DefinedTermBase specimenScanTerm = getTermService().load(UUID.fromString("acda15be-c0e2-4ea8-8783-b9b0c4ad7f03"));
406			if(specimenScanTerm instanceof DefinedTerm){
407			mediaSpecimen.setKindOfUnit((DefinedTerm) specimenScanTerm);
408			}
409	266bc333	Patric Plitzner	DerivationEvent derivationEvent = DerivationEvent.NewInstance(DerivationEventType.PREPARATION());
410			derivationEvent.addDerivative(mediaSpecimen);
411			derivedUnitFacade.innerDerivedUnit().addDerivationEvent(derivationEvent);
412			}
413
414	1d3f5c87	p.kelbert	} catch (MalformedURLException e) {
415			// TODO Auto-generated catch block
416			e.printStackTrace();
417			}
418
419			}
420			}
421
422			// /*
423			// * merge AND STORE DATA
424			// */
425			// getTermService().saveOrUpdate(areaCountry);// TODO save area sooner
426			//
427			// for (NamedArea area : otherAreas) {
428			// getTermService().saveOrUpdate(area);// merge it sooner (foreach area)
429			// }
430	1de96f38	p.kelbert
431	f96df645	Andreas Müller	save(unitsGatheringEvent.getLocality(), state);
432	1de96f38	p.kelbert
433	1d3f5c87	p.kelbert	// handle collection data
434	f96df645	Andreas Müller	setCollectionData(state, derivedUnitFacade);
435	1d3f5c87	p.kelbert
436	1de96f38	p.kelbert	//Reference stuff
437			SpecimenUserInteraction sui = state.getConfig().getSpecimenUserInteraction();
438			Map<String,OriginalSourceBase<?>> sourceMap = new HashMap<String, OriginalSourceBase<?>>();
439
440	64260466	Patrick Plitzner	state.getDataHolder().docSources = new ArrayList<String>();
441			for (String[] fullReference : state.getDataHolder().referenceList) {
442	1de96f38	p.kelbert	String strReference=fullReference[0];
443			String citationDetail = fullReference[1];
444			String citationURL = fullReference[2];
445
446			if (!citationURL.isEmpty()) {
447			citationDetail+=", "+citationURL;
448			}
449
450	5131af90	Patric Plitzner	Reference<?> reference;
451	64260466	Patrick Plitzner	if(strReference.equals(state.getRef().getTitleCache())){
452			reference = state.getRef();
453	5131af90	Patric Plitzner	}
454			else{
455			reference = ReferenceFactory.newGeneric();
456			reference.setTitle(strReference);
457			}
458	1de96f38	p.kelbert
459	cb82395b	p.kelbert	IdentifiableSource sour = getIdentifiableSource(reference,citationDetail);
460
461	1de96f38	p.kelbert	try{
462	9196930e	p.kelbert	if (sour.getCitation() != null){
463	fd1ce664	p.kelbert	if(StringUtils.isNotBlank(sour.getCitationMicroReference())) {
464	64260466	Patrick Plitzner	state.getDataHolder().docSources.add(sour.getCitation().getTitleCache()+ "---"+sour.getCitationMicroReference());
465	fd1ce664	p.kelbert	} else {
466	64260466	Patrick Plitzner	state.getDataHolder().docSources.add(sour.getCitation().getTitleCache());
467	fd1ce664	p.kelbert	}
468	cb82395b	p.kelbert	}
469	1de96f38	p.kelbert	}catch(Exception e){
470			logger.warn("oups");
471			}
472			reference.addSource(sour);
473	fbf3b7d2	Patric Plitzner	save(reference, state);
474	1de96f38	p.kelbert	}
475
476	fdf9eec2	Andreas Müller	List<IdentifiableSource> issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
477			List<DescriptionElementSource> issTmp2 = getCommonService().list(DescriptionElementSource.class, null, null, null, null);
478	1de96f38	p.kelbert
479			Set<OriginalSourceBase> osbSet = new HashSet<OriginalSourceBase>();
480			if(issTmp2!=null) {
481			osbSet.addAll(issTmp2);
482			}
483			if(issTmp!=null) {
484			osbSet.addAll(issTmp);
485			}
486
487	888abfcf	Patric Plitzner	addToSourceMap(sourceMap, osbSet);
488	1de96f38	p.kelbert
489			if( state.getConfig().isInteractWithUser()){
490			List<OriginalSourceBase<?>>sources=null;
491	64260466	Patrick Plitzner	if(!state.isDerivedUnitSourcesSet()){
492			sources= sui.askForSource(sourceMap, "the unit itself","",getReferenceService(), state.getDataHolder().docSources);
493			state.setDerivedUnitSources(sources);
494			state.setDerivedUnitSourcesSet(true);
495	1de96f38	p.kelbert	}
496			else{
497	64260466	Patrick Plitzner	sources=state.getDerivedUnitSources();
498	1de96f38	p.kelbert	}
499	fd1ce664	p.kelbert	// System.out.println("nb sources: "+sources.size());
500			// System.out.println("derivedunitfacade : "+derivedUnitFacade.getTitleCache());
501	1de96f38	p.kelbert	for (OriginalSourceBase<?> sour:sources){
502			if(sour.isInstanceOf(IdentifiableSource.class)){
503	cb82395b	p.kelbert	if(sourceNotLinkedToElement(derivedUnitFacade,sour)) {
504	fd1ce664	p.kelbert	// System.out.println("add source to derivedunitfacade1 "+derivedUnitFacade.getTitleCache());
505	cb82395b	p.kelbert	derivedUnitFacade.addSource((IdentifiableSource)sour.clone());
506			}
507	1de96f38	p.kelbert	}else{
508	cb82395b	p.kelbert	if(sourceNotLinkedToElement(derivedUnitFacade,sour)) {
509	fd1ce664	p.kelbert	// System.out.println("add source to derivedunitfacade2 "+derivedUnitFacade.getTitleCache());
510	cb82395b	p.kelbert	derivedUnitFacade.addSource(OriginalSourceType.Import,sour.getCitation(),sour.getCitationMicroReference(), ioName);
511			}
512	1de96f38	p.kelbert	}
513			}
514			}else{
515			for (OriginalSourceBase<?> sr : sourceMap.values()){
516			if(sr.isInstanceOf(IdentifiableSource.class)){
517	cb82395b	p.kelbert	if(sourceNotLinkedToElement(derivedUnitFacade,sr)) {
518	fd1ce664	p.kelbert	// System.out.println("add source to derivedunitfacade3 "+derivedUnitFacade.getTitleCache());
519	cb82395b	p.kelbert	derivedUnitFacade.addSource((IdentifiableSource)sr.clone());
520			}
521	1de96f38	p.kelbert	}else{
522	cb82395b	p.kelbert	if(sourceNotLinkedToElement(derivedUnitFacade,sr)) {
523	fd1ce664	p.kelbert	// System.out.println("add source to derivedunitfacade4 "+derivedUnitFacade.getTitleCache());
524	cb82395b	p.kelbert	derivedUnitFacade.addSource(OriginalSourceType.Import,sr.getCitation(),sr.getCitationMicroReference(), ioName);
525			}
526	1de96f38	p.kelbert	}
527			}
528			}
529
530	64260466	Patrick Plitzner	save(state.getDerivedUnitBase(), state);
531	1de96f38	p.kelbert
532	1d3f5c87	p.kelbert	// handle identifications
533			handleIdentifications(state, derivedUnitFacade);
534
535			if(DEBUG) {
536			logger.info("saved ABCD specimen ...");
537			}
538
539			} catch (Exception e) {
540			logger.warn("Error when reading record!!");
541			e.printStackTrace();
542			state.setUnsuccessfull();
543			}
544
545			return;
546			}
547
548	207d5d50	Patric Plitzner	private SpecimenOrObservationBase findExistingSpecimen(String unitId, Abcd206ImportState state){
549			ICdmApplicationConfiguration cdmAppController = state.getConfig().getCdmAppController();
550			if(cdmAppController==null){
551			cdmAppController = this;
552			}
553			FindOccurrencesConfigurator config = new FindOccurrencesConfigurator();
554			config.setSignificantIdentifier(unitId);
555			Pager<SpecimenOrObservationBase> existingSpecimens = cdmAppController.getOccurrenceService().findByTitle(config);
556			if(!existingSpecimens.getRecords().isEmpty()){
557			if(existingSpecimens.getRecords().size()==1){
558			return existingSpecimens.getRecords().iterator().next();
559			}
560			}
561			return null;
562			}
563
564	888abfcf	Patric Plitzner	/**
565			* @param sourceMap
566			* @param osbSet
567			*/
568			private void addToSourceMap(Map<String, OriginalSourceBase<?>> sourceMap, Set<OriginalSourceBase> osbSet) {
569			for( OriginalSourceBase<?> osb:osbSet) {
570			if(osb.getCitation()!=null && osb.getCitationMicroReference() !=null && !osb.getCitationMicroReference().isEmpty()) {
571			try{
572			sourceMap.put(osb.getCitation().getTitleCache()+ "---"+osb.getCitationMicroReference(),osb);
573			}catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
574			} else if(osb.getCitation()!=null){
575			try{
576			sourceMap.put(osb.getCitation().getTitleCache(),osb);
577			}catch(NullPointerException e){logger.warn("null pointer problem (no ref?) with "+osb);}
578			}
579			}
580			}
581
582	cb82395b	p.kelbert	/**
583			* @param derivedUnitFacade
584			* @param sour
585			* @return
586			*/
587			private boolean sourceNotLinkedToElement(DerivedUnitFacade derivedUnitFacade, OriginalSourceBase<?> source) {
588			Set<IdentifiableSource> linkedSources = derivedUnitFacade.getSources();
589			for (IdentifiableSource is:linkedSources){
590	6490c596	p.kelbert	if (is.getCitation()!=null && source.getCitation()!=null &&
591			is.getCitation().getTitleCache().equalsIgnoreCase(source.getCitation().getTitleCache())){
592	41ba5f3f	p.kelbert	String isDetail = is.getCitationMicroReference();
593	6490c596	p.kelbert	if ((StringUtils.isBlank(isDetail) && StringUtils.isBlank(source.getCitationMicroReference()))
594			\|\| (isDetail != null && isDetail.equalsIgnoreCase(source.getCitationMicroReference())) ) {
595	41ba5f3f	p.kelbert	return false;
596			}
597	cb82395b	p.kelbert	}
598			}
599			return true;
600			}
601
602			/**
603			* @param reference
604			* @param citationDetail
605			* @return
606			*/
607	6d5513e4	Andreas Müller	//FIXME this method is highly critical, because
608			// * it will have serious performance and memory problems with large databases
609			// (databases may easily have >1 Mio source records)
610			// * it does not make sense to search for existing sources and then clone them
611	9196930e	p.kelbert	// we need to search for existing references instead and use them (if exist)
612	6d5513e4	Andreas Müller	// for our new source.
613	cb82395b	p.kelbert	private IdentifiableSource getIdentifiableSource(Reference<?> reference, String citationDetail) {
614
615	fdf9eec2	Andreas Müller	List<IdentifiableSource> issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
616	cb82395b	p.kelbert
617	9196930e	p.kelbert
618	6d5513e4	Andreas Müller	if (reference != null){
619	fd1ce664	p.kelbert	try {
620			for (OriginalSourceBase<?> osb: issTmp){
621	927fd98d	Patric Plitzner	if (osb.getCitation() != null && osb.getCitation().getTitleCache().equalsIgnoreCase(reference.getTitleCache())){
622	fd1ce664	p.kelbert	String osbDetail = osb.getCitationMicroReference();
623			if ((StringUtils.isBlank(osbDetail) && StringUtils.isBlank(citationDetail))
624			\|\| (osbDetail != null && osbDetail.equalsIgnoreCase(citationDetail)) ) {
625			// System.out.println("REFERENCE FOUND RETURN EXISTING SOURCE");
626			return (IdentifiableSource) osb.clone();
627			}
628			}
629			}
630			} catch (CloneNotSupportedException e) {
631			throw new RuntimeException(e);
632			} catch (Exception e1){
633			e1.printStackTrace();
634			}
635	cb82395b	p.kelbert	}
636	9196930e	p.kelbert
637	cb82395b	p.kelbert	IdentifiableSource sour = IdentifiableSource.NewInstance(OriginalSourceType.Import,null,null, reference,citationDetail);
638			return sour;
639			}
640
641	fd1ce664	p.kelbert	// /**
642			// * @param reference
643			// * @param citationDetail
644			// * @return
645			// */
646			// private DescriptionElementSource getDescriptionSource(Reference<?> reference, String citationDetail) {
647			//
648			// List<OriginalSourceBase> issTmp2 = getCommonService().list(DescriptionElementSource.class, null, null, null, null);
649			//
650			// try {
651			// for (OriginalSourceBase<?> osb:issTmp2){
652			// if (osb.getCitation().equals(reference) && osb.getCitationMicroReference().equalsIgnoreCase(citationDetail)) {
653			// return (DescriptionElementSource) osb.clone();
654			// }
655			// }
656			// } catch (CloneNotSupportedException e) {
657			// // TODO Auto-generated catch block
658			// e.printStackTrace();
659			// }
660			//
661			// DescriptionElementSource sour = DescriptionElementSource.NewInstance(OriginalSourceType.Import,null,null, reference,citationDetail);
662			// return sour;
663			// }
664	cb82395b	p.kelbert
665
666	1d3f5c87	p.kelbert	/**
667	1de96f38	p.kelbert	* Very fast and dirty implementation to allow handling of transient objects as described in
668	f96df645	Andreas Müller	* https://dev.e-taxonomy.eu/trac/ticket/3726
669	1de96f38	p.kelbert	*
670	f96df645	Andreas Müller	* Not yet complete.
671	1de96f38	p.kelbert	*
672	f96df645	Andreas Müller	* @param cdmBase
673			* @param state
674			*/
675			private void save(CdmBase cdmBase, Abcd206ImportState state) {
676	0092a4a5	Cherian Mathew	ICdmApplicationConfiguration cdmRepository = state.getConfig().getCdmAppController();
677	1de96f38	p.kelbert	if (cdmRepository == null){
678			cdmRepository = this;
679			}
680
681			if (cdmBase.isInstanceOf(LanguageString.class)){
682			cdmRepository.getTermService().saveLanguageData(CdmBase.deproxy(cdmBase, LanguageString.class));
683			}else if (cdmBase.isInstanceOf(SpecimenOrObservationBase.class)){
684			cdmRepository.getOccurrenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, SpecimenOrObservationBase.class));
685			}else if (cdmBase.isInstanceOf(Reference.class)){
686			cdmRepository.getReferenceService().saveOrUpdate(CdmBase.deproxy(cdmBase, Reference.class));
687			}else if (cdmBase.isInstanceOf(Classification.class)){
688			cdmRepository.getClassificationService().saveOrUpdate(CdmBase.deproxy(cdmBase, Classification.class));
689			}else if (cdmBase.isInstanceOf(AgentBase.class)){
690			cdmRepository.getAgentService().saveOrUpdate(CdmBase.deproxy(cdmBase, AgentBase.class));
691			}else if (cdmBase.isInstanceOf(Collection.class)){
692			cdmRepository.getCollectionService().saveOrUpdate(CdmBase.deproxy(cdmBase, Collection.class));
693			}else if (cdmBase.isInstanceOf(DescriptionBase.class)){
694			cdmRepository.getDescriptionService().saveOrUpdate(CdmBase.deproxy(cdmBase, DescriptionBase.class));
695			}else if (cdmBase.isInstanceOf(TaxonBase.class)){
696			cdmRepository.getTaxonService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonBase.class));
697			}else if (cdmBase.isInstanceOf(TaxonNameBase.class)){
698			cdmRepository.getNameService().saveOrUpdate(CdmBase.deproxy(cdmBase, TaxonNameBase.class));
699			}else{
700			throw new IllegalArgumentException("Class not supported in save method: " + CdmBase.deproxy(cdmBase, CdmBase.class).getClass().getSimpleName());
701			}
702
703	f96df645	Andreas Müller	}
704
705	1de96f38	p.kelbert	/**
706	1d3f5c87	p.kelbert	* setCollectionData : store the collection object into the
707			* derivedUnitFacade
708			*
709	f96df645	Andreas Müller	* @param state
710	1d3f5c87	p.kelbert	*/
711	f96df645	Andreas Müller	private void setCollectionData(Abcd206ImportState state, DerivedUnitFacade derivedUnitFacade) {
712	fbf34d08	Patric Plitzner	Abcd206ImportConfigurator config = state.getConfig();
713	64260466	Patrick Plitzner	AbcdImportUtility.setUnitID(derivedUnitFacade.innerDerivedUnit(), state.getDataHolder().unitID, config);
714	2bc696d1	Patrick Plitzner	if(!config.isMapUnitIdToAccessionNumber()){
715	64260466	Patrick Plitzner	derivedUnitFacade.setAccessionNumber(NB(state.getDataHolder().accessionNumber));
716	fbf34d08	Patric Plitzner	}
717	64260466	Patrick Plitzner	// derivedUnitFacade.setCollectorsNumber(NB(state.getDataHolder().collectorsNumber));
718	1d3f5c87	p.kelbert
719			/*
720			* INSTITUTION & COLLECTION
721			*/
722			// manage institution
723	64260466	Patrick Plitzner	Institution institution = this.getInstitution(NB(state.getDataHolder().institutionCode), state);
724	1d3f5c87	p.kelbert	// manage collection
725	64260466	Patrick Plitzner	Collection collection = this.getCollection(institution, NB(state.getDataHolder().collectionCode), state);
726	1d3f5c87	p.kelbert	// link specimen & collection
727			derivedUnitFacade.setCollection(collection);
728			}
729
730			/**
731			* getFacade : get the DerivedUnitFacade based on the recordBasis
732	64260466	Patrick Plitzner	* @param state
733	1d3f5c87	p.kelbert	*
734			* @return DerivedUnitFacade
735			*/
736	64260466	Patrick Plitzner	private DerivedUnitFacade getFacade(Abcd206ImportState state) {
737	1de96f38	p.kelbert	if(DEBUG) {
738			logger.info("getFacade()");
739			}
740			SpecimenOrObservationType type = null;
741
742			// create specimen
743	64260466	Patrick Plitzner	if (NB((state.getDataHolder().recordBasis)) != null) {
744			if (state.getDataHolder().recordBasis.toLowerCase().startsWith("s") \|\| state.getDataHolder().recordBasis.toLowerCase().indexOf("specimen")>-1) {// specimen
745	1de96f38	p.kelbert	type = SpecimenOrObservationType.PreservedSpecimen;
746			}
747	64260466	Patrick Plitzner	if (state.getDataHolder().recordBasis.toLowerCase().startsWith("o") \|\|state.getDataHolder().recordBasis.toLowerCase().indexOf("observation")>-1 ) {
748	1de96f38	p.kelbert	type = SpecimenOrObservationType.Observation;
749			}
750	64260466	Patrick Plitzner	if (state.getDataHolder().recordBasis.toLowerCase().indexOf("fossil")>-1){
751	1de96f38	p.kelbert	type = SpecimenOrObservationType.Fossil;
752			}
753	64260466	Patrick Plitzner	if (state.getDataHolder().recordBasis.toLowerCase().indexOf("living")>-1) {
754	1de96f38	p.kelbert	type = SpecimenOrObservationType.LivingSpecimen;
755			}
756			if (type == null) {
757	64260466	Patrick Plitzner	logger.info("The basis of record does not seem to be known: " + state.getDataHolder().recordBasis);
758	1de96f38	p.kelbert	type = SpecimenOrObservationType.DerivedUnit;
759			}
760			// TODO fossils?
761			} else {
762			logger.info("The basis of record is null");
763			type = SpecimenOrObservationType.DerivedUnit;
764			}
765			DerivedUnitFacade derivedUnitFacade = DerivedUnitFacade.NewInstance(type);
766			return derivedUnitFacade;
767	1d3f5c87	p.kelbert	}
768
769	64260466	Patrick Plitzner	private void getCollectorsFromXML(Element root, Abcd206XMLFieldGetter abcdFieldGetter, Abcd206ImportState state) {
770	1d3f5c87	p.kelbert	NodeList group;
771
772			group = root.getChildNodes();
773			for (int i = 0; i < group.getLength(); i++) {
774			if (group.item(i).getNodeName().equals(prefix + "Identifications")) {
775			group = group.item(i).getChildNodes();
776			break;
777			}
778			}
779	64260466	Patrick Plitzner	state.getDataHolder().gatheringAgentList = new ArrayList<String>();
780			state.getDataHolder().gatheringTeamList = new ArrayList<String>();
781	1d3f5c87	p.kelbert	abcdFieldGetter.getType(root);
782			abcdFieldGetter.getGatheringPeople(root);
783			}
784
785			/**
786			* Store the unit's properties into variables Look which unit is the
787			* preferred one Look what kind of name it is supposed to be, for the
788			* parsing (Botanical, Zoological)
789	927fd98d	Patric Plitzner	* @param state
790	1d3f5c87	p.kelbert	*
791			* @param racine: the root node for a single unit
792			*/
793	927fd98d	Patric Plitzner	private void setUnitPropertiesXML(Element root, Abcd206XMLFieldGetter abcdFieldGetter, Abcd206ImportState state) {
794	1d3f5c87	p.kelbert	try {
795			NodeList group;
796
797			group = root.getChildNodes();
798			for (int i = 0; i < group.getLength(); i++) {
799			if (group.item(i).getNodeName().equals(prefix + "Identifications")) {
800			group = group.item(i).getChildNodes();
801			break;
802			}
803			}
804	64260466	Patrick Plitzner	state.getDataHolder().identificationList = new ArrayList<Identification>();
805			state.getDataHolder().statusList = new ArrayList<SpecimenTypeDesignationStatus>();
806			state.getDataHolder().atomisedIdentificationList = new ArrayList<HashMap<String, String>>();
807			state.getDataHolder().referenceList = new ArrayList<String[]>();
808			state.getDataHolder().multimediaObjects = new ArrayList<String>();
809	1d3f5c87	p.kelbert
810			abcdFieldGetter.getScientificNames(group);
811			abcdFieldGetter.getType(root);
812
813			if(DEBUG) {
814	64260466	Patrick Plitzner	logger.info("this.identificationList "+state.getDataHolder().identificationList.toString());
815	1d3f5c87	p.kelbert	}
816			abcdFieldGetter.getIDs(root);
817			abcdFieldGetter.getRecordBasis(root);
818	207d5d50	Patric Plitzner	abcdFieldGetter.getKindOfUnit(root);
819	1d3f5c87	p.kelbert	abcdFieldGetter.getMultimedia(root);
820			abcdFieldGetter.getNumbers(root);
821	927fd98d	Patric Plitzner	abcdFieldGetter.getGeolocation(root, state);
822	1d3f5c87	p.kelbert	abcdFieldGetter.getGatheringPeople(root);
823	0fd39e45	Patric Plitzner	abcdFieldGetter.getGatheringDate(root);
824			abcdFieldGetter.getGatheringElevation(root);
825	207d5d50	Patric Plitzner	abcdFieldGetter.getAssociatedUnitIds(root);
826	1d3f5c87	p.kelbert	boolean referencefound = abcdFieldGetter.getReferences(root);
827			if (!referencefound) {
828	64260466	Patrick Plitzner	String[]a = {state.getRef().getTitleCache(),"",""};
829			state.getDataHolder().referenceList.add(a);
830	1d3f5c87	p.kelbert	}
831
832			} catch (Exception e) {
833			logger.info("Error occured while parsing XML file" + e);
834			}
835			}
836
837			/**
838			* Look if the Institution does already exist
839			* @param institutionCode: a string with the institutioncode
840			* @param config : the configurator
841			* @return the Institution (existing or new)
842			*/
843	f96df645	Andreas Müller	private Institution getInstitution(String institutionCode, Abcd206ImportState state) {
844	1d3f5c87	p.kelbert	Institution institution=null;
845	fdf9eec2	Andreas Müller	List<Institution> institutions;
846	1d3f5c87	p.kelbert	try {
847			institutions = getAgentService().list(Institution.class, null, null, null, null);
848			} catch (Exception e) {
849	fdf9eec2	Andreas Müller	institutions = new ArrayList<Institution>();
850	1d3f5c87	p.kelbert	logger.warn(e);
851			}
852	f96df645	Andreas Müller	if (institutions.size() > 0 && state.getConfig().isReUseExistingMetadata()) {
853	fdf9eec2	Andreas Müller	for (Institution institut:institutions){
854	1de96f38	p.kelbert	try{
855			if (institut.getCode().equalsIgnoreCase(institutionCode)) {
856			institution=institut;
857			}
858			}catch(Exception e){logger.warn("no institution code in the db");}
859	1d3f5c87	p.kelbert	}
860			}
861			if(DEBUG) {
862	1de96f38	p.kelbert	if(institution !=null) {
863			logger.info("getinstitution " + institution.toString());
864			}
865	1d3f5c87	p.kelbert	}
866			if (institution == null){
867			// create institution
868			institution = Institution.NewInstance();
869			institution.setCode(institutionCode);
870	fdf9eec2	Andreas Müller	institution.setTitleCache(institutionCode, true);
871	1d3f5c87	p.kelbert	}
872	f96df645	Andreas Müller	save(institution, state);
873	1d3f5c87	p.kelbert	return institution;
874			}
875
876			/**
877			* Look if the Collection does already exist
878			* @param collectionCode
879			* @param collectionCode: a string
880			* @param config : the configurator
881			* @return the Collection (existing or new)
882			*/
883	f96df645	Andreas Müller	private Collection getCollection(Institution institution, String collectionCode, Abcd206ImportState state) {
884	1d3f5c87	p.kelbert	Collection collection = null;
885			List<Collection> collections;
886			try {
887			collections = getCollectionService().list(Collection.class, null, null, null, null);
888			} catch (Exception e) {
889			collections = new ArrayList<Collection>();
890			}
891	f96df645	Andreas Müller	if (collections.size() > 0 && state.getConfig().isReUseExistingMetadata()) {
892	1d3f5c87	p.kelbert	for (Collection coll:collections){
893			if (coll.getInstitute() != null) {
894			if (coll.getCode().equalsIgnoreCase(collectionCode) && coll.getInstitute().equals(institution)) {
895			collection=coll;
896			}
897			}
898			}
899			}
900
901			if(collection == null){
902			collection =Collection.NewInstance();
903			collection.setCode(collectionCode);
904			collection.setInstitute(institution);
905			collection.setTitleCache(collectionCode);
906			}
907	f96df645	Andreas Müller	save(collection, state);
908	1d3f5c87	p.kelbert	return collection;
909			}
910
911
912			/**
913			* join DeterminationEvent to the Taxon Object
914			* @param state : the ABCD import state
915			* @param taxon: the current Taxon
916			* @param preferredFlag :if the current name is preferred
917			* @param derivedFacade : the derived Unit Facade
918			*/
919			@SuppressWarnings("rawtypes")
920			private void linkDeterminationEvent(Abcd206ImportState state, Taxon taxon, boolean preferredFlag, DerivedUnitFacade derivedFacade) {
921			Abcd206ImportConfigurator config = state.getConfig();
922			if(DEBUG){
923			logger.info("start linkdetermination with taxon:" + taxon.getUuid()+", "+taxon);
924			}
925
926			DeterminationEvent determinationEvent = DeterminationEvent.NewInstance();
927	ce8437b0	Patric Plitzner	determinationEvent.setTaxonName(taxon.getName());
928	1d3f5c87	p.kelbert	determinationEvent.setPreferredFlag(preferredFlag);
929
930	64260466	Patrick Plitzner	determinationEvent.setIdentifiedUnit(state.getDerivedUnitBase());
931			state.getDerivedUnitBase().addDetermination(determinationEvent);
932	1d3f5c87	p.kelbert
933	4134e828	Patric Plitzner	if(DEBUG){
934	64260466	Patrick Plitzner	logger.info("NB TYPES INFO: "+ state.getDataHolder().statusList.size());
935	4134e828	Patric Plitzner	}
936	64260466	Patrick Plitzner	for (SpecimenTypeDesignationStatus specimenTypeDesignationstatus : state.getDataHolder().statusList) {
937	4134e828	Patric Plitzner	if (specimenTypeDesignationstatus != null) {
938			if(DEBUG){
939			logger.info("specimenTypeDesignationstatus :"+ specimenTypeDesignationstatus);
940			}
941	1d3f5c87	p.kelbert
942	4134e828	Patric Plitzner	ICdmApplicationConfiguration cdmAppController = config.getCdmAppController();
943			if(cdmAppController == null){
944			cdmAppController = this;
945	1d3f5c87	p.kelbert	}
946	4134e828	Patric Plitzner	specimenTypeDesignationstatus = (SpecimenTypeDesignationStatus) cdmAppController.getTermService().find(specimenTypeDesignationstatus.getUuid());
947			//Designation
948			TaxonNameBase<?,?> name = taxon.getName();
949			SpecimenTypeDesignation designation = SpecimenTypeDesignation.NewInstance();
950
951			designation.setTypeStatus(specimenTypeDesignationstatus);
952	64260466	Patrick Plitzner	designation.setTypeSpecimen(state.getDerivedUnitBase());
953	4134e828	Patric Plitzner	name.addTypeDesignation(designation, true);
954	1d3f5c87	p.kelbert	}
955			}
956
957	64260466	Patrick Plitzner	for (String[] fullReference : state.getDataHolder().referenceList) {
958	4134e828	Patric Plitzner	List<Reference> references = getReferenceService().list(Reference.class, null, null, null, null);
959
960			String strReference=fullReference[0];
961			String citationDetail = fullReference[1];
962			String citationURL = fullReference[2];
963
964			if (isNotBlank(strReference)){
965			Reference<?> reference = null;
966			for (Reference<?> refe: references) {
967			if (refe.getTitleCache().equalsIgnoreCase(strReference)) {
968			reference =refe;
969			break;
970	1d3f5c87	p.kelbert	}
971			}
972	4134e828	Patric Plitzner	if (reference ==null){
973			reference = ReferenceFactory.newGeneric();
974			reference.setTitleCache(strReference, true);
975			save(reference, state);
976			}
977			determinationEvent.addReference(reference);
978	b4426234	Patric Plitzner	}
979	1d3f5c87	p.kelbert	}
980	64260466	Patrick Plitzner	save(state.getDerivedUnitBase(), state);
981	1d3f5c87	p.kelbert
982	8d425e15	Patric Plitzner	if (config.isAddIndividualsAssociationsSuchAsSpecimenAndObservations() && preferredFlag) {
983			//do not add IndividualsAssociation to non-preferred taxa
984	1d3f5c87	p.kelbert	if(DEBUG){
985			logger.info("isDoCreateIndividualsAssociations");
986			}
987
988			makeIndividualsAssociation(state, taxon, determinationEvent);
989	4134e828	Patric Plitzner
990			if(state.getConfig().isDeterminationOnFieldUnitLevel()){
991			DeterminationEvent fieldUnitDeterminationEvent = DeterminationEvent.NewInstance();
992	ce8437b0	Patric Plitzner	fieldUnitDeterminationEvent.setTaxonName(determinationEvent.getTaxonName());
993	4134e828	Patric Plitzner	fieldUnitDeterminationEvent.setPreferredFlag(determinationEvent.getPreferredFlag());
994	64260466	Patrick Plitzner	fieldUnitDeterminationEvent.setIdentifiedUnit(state.getFieldUnit());
995	4134e828	Patric Plitzner	Set<Reference> references = determinationEvent.getReferences();
996			for (Reference reference : references) {
997			fieldUnitDeterminationEvent.addReference(reference);
998			}
999	64260466	Patrick Plitzner	state.getFieldUnit().addDetermination(fieldUnitDeterminationEvent);
1000	4134e828	Patric Plitzner	}
1001
1002	64260466	Patrick Plitzner	save(state.getDerivedUnitBase(), state);
1003	1d3f5c87	p.kelbert	}
1004			}
1005
1006			/**
1007			* create and link each association (specimen, observation..) to the accepted taxon
1008			* @param state : the ABCD import state
1009			* @param taxon: the current Taxon
1010			* @param determinationEvent:the determinationevent
1011			*/
1012			private void makeIndividualsAssociation(Abcd206ImportState state, Taxon taxon, DeterminationEvent determinationEvent) {
1013	1de96f38	p.kelbert	SpecimenUserInteraction sui = state.getConfig().getSpecimenUserInteraction();
1014
1015	1d3f5c87	p.kelbert	if (DEBUG) {
1016	e2e537ff	Patric Plitzner	logger.info("MAKE INDIVIDUALS ASSOCIATION");
1017	1d3f5c87	p.kelbert	}
1018
1019			TaxonDescription taxonDescription = null;
1020			Set<TaxonDescription> descriptions= taxon.getDescriptions();
1021	1de96f38	p.kelbert	if (state.getConfig().isInteractWithUser()){
1022	64260466	Patrick Plitzner	if(!state.isDescriptionGroupSet()){
1023	fd1ce664	p.kelbert	taxonDescription = sui.askForDescriptionGroup(descriptions);
1024	64260466	Patrick Plitzner	state.setDescriptionGroup(taxonDescription);
1025			state.setDescriptionGroupSet(true);
1026	fd1ce664	p.kelbert	}else{
1027	64260466	Patrick Plitzner	taxonDescription=state.getDescriptionGroup();
1028	fd1ce664	p.kelbert	}
1029	1de96f38	p.kelbert	} else {
1030			for (TaxonDescription description : descriptions){
1031			Set<IdentifiableSource> sources = description.getTaxon().getSources();
1032			sources.addAll(description.getSources());
1033			for (IdentifiableSource source:sources){
1034	64260466	Patrick Plitzner	if(state.getRef().equals(source.getCitation())) {
1035	1de96f38	p.kelbert	taxonDescription = description;
1036			}
1037	1d3f5c87	p.kelbert	}
1038			}
1039			}
1040			if (taxonDescription == null){
1041			taxonDescription = TaxonDescription.NewInstance(taxon, false);
1042	64260466	Patrick Plitzner	if(sourceNotLinkedToElement(taxonDescription,state.getRef(),null)) {
1043			taxonDescription.addSource(OriginalSourceType.Import, null, null, state.getRef(), null);
1044	cb82395b	p.kelbert	}
1045	64260466	Patrick Plitzner	state.setDescriptionGroup(taxonDescription);
1046	1d3f5c87	p.kelbert	taxon.addDescription(taxonDescription);
1047			}
1048
1049	1de96f38	p.kelbert	//PREPARE REFERENCE QUESTIONS
1050
1051			Map<String,OriginalSourceBase<?>> sourceMap = new HashMap<String, OriginalSourceBase<?>>();
1052
1053	fdf9eec2	Andreas Müller	List<IdentifiableSource> issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
1054			List<DescriptionElementSource> issTmp2 = getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1055	1de96f38	p.kelbert
1056			Set<OriginalSourceBase> osbSet = new HashSet<OriginalSourceBase>();
1057			if(issTmp2!=null) {
1058			osbSet.addAll(issTmp2);
1059			}
1060			if(issTmp!=null) {
1061			osbSet.addAll(issTmp);
1062			}
1063
1064
1065	888abfcf	Patric Plitzner	addToSourceMap(sourceMap, osbSet);
1066	1de96f38	p.kelbert
1067			if (state.getConfig().isInteractWithUser()){
1068			List<OriginalSourceBase<?>> res = null;
1069	64260466	Patrick Plitzner	if(!state.isDescriptionSourcesSet()){
1070	cb82395b	p.kelbert	res = sui.askForSource(sourceMap, "the description group ("+taxon+")",
1071	64260466	Patrick Plitzner	"The current reference is "+state.getRef().getTitleCache(),getReferenceService(), state.getDataHolder().docSources);
1072			state.setDescriptionRefs(res);
1073			state.setDescriptionSourcesSet(true);
1074	1de96f38	p.kelbert	}
1075			else{
1076	64260466	Patrick Plitzner	res=state.getDescriptionRefs();
1077	1de96f38	p.kelbert	}
1078			if(res !=null) {
1079			for (OriginalSourceBase<?> sour:res){
1080			if(sour.isInstanceOf(IdentifiableSource.class)){
1081			try {
1082	cb82395b	p.kelbert	if(sourceNotLinkedToElement(taxonDescription,sour)) {
1083			taxonDescription.addSource((IdentifiableSource)sour.clone());
1084			}
1085	1de96f38	p.kelbert	} catch (CloneNotSupportedException e) {
1086			logger.warn("no cloning?");
1087			}
1088			}else{
1089	cb82395b	p.kelbert	if(sourceNotLinkedToElement(taxonDescription,sour)) {
1090			taxonDescription.addSource(OriginalSourceType.Import,null, null, sour.getCitation(),sour.getCitationMicroReference());
1091			}
1092	1de96f38	p.kelbert	}
1093			}
1094			}
1095			}
1096			else {
1097	64260466	Patrick Plitzner	if(sourceNotLinkedToElement(taxonDescription,state.getRef(),null)) {
1098			taxonDescription.addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1099	fd1ce664	p.kelbert	}
1100	cb82395b	p.kelbert	}
1101	64260466	Patrick Plitzner	state.setDescriptionGroup(taxonDescription);
1102	1d3f5c87	p.kelbert
1103			IndividualsAssociation indAssociation = IndividualsAssociation.NewInstance();
1104	64260466	Patrick Plitzner	Feature feature = makeFeature(state.getDerivedUnitBase());
1105			indAssociation.setAssociatedSpecimenOrObservation(state.getDerivedUnitBase());
1106	1d3f5c87	p.kelbert	indAssociation.setFeature(feature);
1107
1108	1de96f38	p.kelbert	if (state.getConfig().isInteractWithUser()){
1109			sourceMap = new HashMap<String, OriginalSourceBase<?>>();
1110
1111			issTmp = getCommonService().list(IdentifiableSource.class, null, null, null, null);
1112			issTmp2 = getCommonService().list(DescriptionElementSource.class, null, null, null, null);
1113
1114			osbSet = new HashSet<OriginalSourceBase>();
1115			if(issTmp2!=null) {
1116			osbSet.addAll(issTmp2);
1117			}
1118			if(issTmp!=null) {
1119			osbSet.addAll(issTmp);
1120			}
1121
1122
1123	888abfcf	Patric Plitzner	addToSourceMap(sourceMap, osbSet);
1124	1de96f38	p.kelbert
1125			List<OriginalSourceBase<?>> sources =null;
1126	64260466	Patrick Plitzner	if(!state.isAssociationSourcesSet()) {
1127	cb82395b	p.kelbert	sources = sui.askForSource(sourceMap, "descriptive element (association) ",taxon.toString(),
1128	64260466	Patrick Plitzner	getReferenceService(),state.getDataHolder().docSources);
1129			state.setAssociationRefs(sources);
1130			state.setAssociationSourcesSet(true);
1131	1de96f38	p.kelbert	}
1132			else{
1133	64260466	Patrick Plitzner	sources=state.getAssociationRefs();
1134	1de96f38	p.kelbert	}
1135			if(sources !=null) {
1136			for (OriginalSourceBase<?> source: sources) {
1137			if(source !=null) {
1138			if(source.isInstanceOf(DescriptionElementSource.class)){
1139			try {
1140	cb82395b	p.kelbert	if(sourceNotLinkedToElement(indAssociation,source)) {
1141			indAssociation.addSource((DescriptionElementSource)source.clone());
1142			}
1143	1de96f38	p.kelbert	} catch (CloneNotSupportedException e) {
1144			logger.warn("clone forbidden?");
1145			}
1146			}else{
1147	cb82395b	p.kelbert	if(sourceNotLinkedToElement(indAssociation,source)) {
1148			indAssociation.addSource(OriginalSourceType.Import,null, null, source.getCitation(),source.getCitationMicroReference());
1149			}
1150			try {
1151	64260466	Patrick Plitzner	if(sourceNotLinkedToElement(state.getDerivedUnitBase(), source)) {
1152			state.getDerivedUnitBase().addSource((IdentifiableSource) source.clone());
1153	cb82395b	p.kelbert	}
1154			} catch (CloneNotSupportedException e) {
1155			// TODO Auto-generated catch block
1156			e.printStackTrace();
1157			}
1158	1de96f38	p.kelbert	}
1159
1160			}
1161			}
1162			}
1163			}else {
1164	64260466	Patrick Plitzner	if(sourceNotLinkedToElement(indAssociation,state.getRef(),null)) {
1165			indAssociation.addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1166	cb82395b	p.kelbert	}
1167	64260466	Patrick Plitzner	if(sourceNotLinkedToElement(state.getDerivedUnitBase(), state.getRef(),null)) {
1168			state.getDerivedUnitBase().addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1169	fd1ce664	p.kelbert	}
1170	1de96f38	p.kelbert	for (Reference<?> citation : determinationEvent.getReferences()) {
1171	cb82395b	p.kelbert	if(sourceNotLinkedToElement(indAssociation,citation,null))
1172	fd1ce664	p.kelbert	{
1173	cb82395b	p.kelbert	indAssociation.addSource(DescriptionElementSource.NewInstance(OriginalSourceType.Import, null, null, citation, null));
1174			}
1175	64260466	Patrick Plitzner	if(sourceNotLinkedToElement(state.getDerivedUnitBase(), state.getRef(),null)) {
1176			state.getDerivedUnitBase().addSource(OriginalSourceType.Import,null, null, state.getRef(), null);
1177	fd1ce664	p.kelbert	}
1178	1de96f38	p.kelbert	}
1179	1d3f5c87	p.kelbert	}
1180
1181			taxonDescription.addElement(indAssociation);
1182
1183	f96df645	Andreas Müller	save(taxonDescription, state);
1184			save(taxon, state);
1185	64260466	Patrick Plitzner	report.addDerivate(state.getDataHolder().unitID, state.getDerivedUnitBase());
1186			report.addIndividualAssociation(taxon, state.getDataHolder().unitID, state.getDerivedUnitBase());
1187	1d3f5c87	p.kelbert	}
1188
1189	cb82395b	p.kelbert	/**
1190			* @param derivedUnitBase2
1191			* @param ref2
1192			* @param object
1193			* @return
1194			*/
1195			private boolean sourceNotLinkedToElement(DerivedUnit derivedUnitBase2, Reference<?> b, String d) {
1196			Set<IdentifiableSource> linkedSources = derivedUnitBase2.getSources();
1197			for (IdentifiableSource is:linkedSources){
1198	d26d6619	Andreas M��ller	Reference<?> a = is.getCitation();
1199	cb82395b	p.kelbert	String c = is.getCitationMicroReference();
1200
1201			boolean refMatch=false;
1202			boolean microMatch=false;
1203
1204			try{
1205			if (a==null && b==null) {
1206			refMatch=true;
1207			}
1208			if (a!=null && b!=null) {
1209			if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1210			refMatch=true;
1211			}
1212			}
1213			}catch(Exception e){}
1214
1215
1216			try{
1217			if (c==null && d==null) {
1218			microMatch=true;
1219			}
1220			if(c!=null && d!=null) {
1221			if(c.equalsIgnoreCase(d)) {
1222			microMatch=true;
1223			}
1224			}
1225			}
1226			catch(Exception e){}
1227
1228			if (microMatch && refMatch) {
1229			return false;
1230			}
1231
1232
1233			}
1234			return true;
1235			}
1236
1237			/**
1238	4134e828	Patric Plitzner	* @param specimen
1239	cb82395b	p.kelbert	* @param source
1240			* @return
1241			*/
1242	4134e828	Patric Plitzner	private boolean sourceNotLinkedToElement(SpecimenOrObservationBase<?> specimen, OriginalSourceBase<?> source) {
1243			Set<IdentifiableSource> linkedSources = specimen.getSources();
1244	cb82395b	p.kelbert	for (IdentifiableSource is:linkedSources){
1245	d26d6619	Andreas M��ller	Reference<?> a = is.getCitation();
1246			Reference<?> b = source.getCitation();
1247	cb82395b	p.kelbert	String c = is.getCitationMicroReference();
1248			String d = source.getCitationMicroReference();
1249
1250			boolean refMatch=false;
1251			boolean microMatch=false;
1252
1253			try{
1254			if (a==null && b==null) {
1255			refMatch=true;
1256			}
1257			if (a!=null && b!=null) {
1258			if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1259			refMatch=true;
1260			}
1261			}
1262			}catch(Exception e){}
1263
1264
1265			try{
1266			if (c==null && d==null) {
1267			microMatch=true;
1268			}
1269			if(c!=null && d!=null) {
1270			if(c.equalsIgnoreCase(d)) {
1271			microMatch=true;
1272			}
1273			}
1274			}
1275			catch(Exception e){}
1276
1277			if (microMatch && refMatch) {
1278			return false;
1279			}
1280
1281
1282			}
1283			return true;
1284			}
1285
1286			/**
1287			* @param indAssociation
1288			* @param ref2
1289			* @param object
1290			* @return
1291			*/
1292			private boolean sourceNotLinkedToElement(IndividualsAssociation indAssociation, Reference<?> a, String d) {
1293			Set<DescriptionElementSource> linkedSources = indAssociation.getSources();
1294			for (DescriptionElementSource is:linkedSources){
1295	d26d6619	Andreas M��ller	Reference<?> b = is.getCitation();
1296	cb82395b	p.kelbert	String c = is.getCitationMicroReference();
1297
1298			boolean refMatch=false;
1299			boolean microMatch=false;
1300
1301			try{
1302			if (a==null && b==null) {
1303			refMatch=true;
1304			}
1305			if (a!=null && b!=null) {
1306			if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1307			refMatch=true;
1308			}
1309			}
1310			}catch(Exception e){}
1311
1312
1313			try{
1314			if (c==null && d==null) {
1315			microMatch=true;
1316			}
1317			if(c!=null && d!=null) {
1318			if(c.equalsIgnoreCase(d)) {
1319			microMatch=true;
1320			}
1321			}
1322			}
1323			catch(Exception e){}
1324
1325			if (microMatch && refMatch) {
1326			return false;
1327			}
1328			}
1329			return true;
1330			}
1331
1332			/**
1333			* @param taxonDescription
1334			* @param ref2
1335			* @param object
1336			* @return
1337			*/
1338			private boolean sourceNotLinkedToElement(TaxonDescription taxonDescription, Reference<?> a, String d) {
1339			Set<IdentifiableSource> linkedSources = taxonDescription.getSources();
1340			for (IdentifiableSource is:linkedSources){
1341	d26d6619	Andreas M��ller	Reference<?> b = is.getCitation();
1342	cb82395b	p.kelbert	String c = is.getCitationMicroReference();
1343
1344			boolean refMatch=false;
1345			boolean microMatch=false;
1346
1347			try{
1348			if (a==null && b==null) {
1349			refMatch=true;
1350			}
1351			if (a!=null && b!=null) {
1352			if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1353			refMatch=true;
1354			}
1355			}
1356			}catch(Exception e){}
1357
1358
1359			try{
1360			if (c==null && d==null) {
1361			microMatch=true;
1362			}
1363			if(c!=null && d!=null) {
1364			if(c.equalsIgnoreCase(d)) {
1365			microMatch=true;
1366			}
1367			}
1368			}
1369			catch(Exception e){}
1370
1371			if (microMatch && refMatch) {
1372			return false;
1373			}
1374			}
1375			return true;
1376			}
1377
1378			/**
1379			* @param indAssociation
1380			* @param source
1381			* @return
1382			*/
1383			private boolean sourceNotLinkedToElement(IndividualsAssociation indAssociation, OriginalSourceBase<?> source) {
1384			Set<DescriptionElementSource> linkedSources = indAssociation.getSources();
1385			for (DescriptionElementSource is:linkedSources){
1386	d26d6619	Andreas M��ller	Reference<?> a = is.getCitation();
1387			Reference<?> b = source.getCitation();
1388	cb82395b	p.kelbert	String c = is.getCitationMicroReference();
1389			String d = source.getCitationMicroReference();
1390
1391			boolean refMatch=false;
1392			boolean microMatch=false;
1393
1394			try{
1395			if (a==null && b==null) {
1396			refMatch=true;
1397			}
1398			if (a!=null && b!=null) {
1399			if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1400			refMatch=true;
1401			}
1402			}
1403			}catch(Exception e){}
1404
1405
1406			try{
1407			if (c==null && d==null) {
1408			microMatch=true;
1409			}
1410			if(c!=null && d!=null) {
1411			if(c.equalsIgnoreCase(d)) {
1412			microMatch=true;
1413			}
1414			}
1415			}
1416			catch(Exception e){}
1417
1418			if (microMatch && refMatch) {
1419			return false;
1420			}
1421			}
1422			return true;
1423			}
1424
1425			/**
1426			* @param taxonDescription
1427			* @param sour
1428			* @return
1429			*/
1430			private boolean sourceNotLinkedToElement(TaxonDescription taxonDescription, OriginalSourceBase<?> sour) {
1431			Set<IdentifiableSource> linkedSources = taxonDescription.getSources();
1432			for (IdentifiableSource is:linkedSources){
1433	d26d6619	Andreas M��ller	Reference<?> a = is.getCitation();
1434			Reference<?> b = sour.getCitation();
1435	cb82395b	p.kelbert	String c = is.getCitationMicroReference();
1436			String d = sour.getCitationMicroReference();
1437
1438			boolean refMatch=false;
1439			boolean microMatch=false;
1440
1441			try{
1442			if (a==null && b==null) {
1443			refMatch=true;
1444			}
1445			if (a!=null && b!=null) {
1446			if (a.getTitleCache().equalsIgnoreCase(b.getTitleCache())) {
1447			refMatch=true;
1448			}
1449			}
1450			}catch(Exception e){}
1451
1452
1453			try{
1454			if (c==null && d==null) {
1455			microMatch=true;
1456			}
1457			if(c!=null && d!=null) {
1458			if(c.equalsIgnoreCase(d)) {
1459			microMatch=true;
1460			}
1461			}
1462			}
1463			catch(Exception e){}
1464
1465			if (microMatch && refMatch) {
1466			return false;
1467			}
1468
1469
1470			}
1471			return true;
1472			}
1473
1474	1d3f5c87	p.kelbert	/**
1475			* look for the Feature object (FieldObs, Specimen,...)
1476			* @param unit : a specimen or obersvation base
1477			* @return the corresponding Feature
1478			*/
1479			private Feature makeFeature(SpecimenOrObservationBase<?> unit) {
1480	1de96f38	p.kelbert	SpecimenOrObservationType type = unit.getRecordBasis();
1481	6490c596	p.kelbert
1482
1483
1484	1de96f38	p.kelbert	if (type.isFeatureObservation()){
1485			return Feature.OBSERVATION();
1486			}else if (type.isFeatureSpecimen()){
1487			return Feature.SPECIMEN();
1488			}else if (type == SpecimenOrObservationType.DerivedUnit){
1489	6490c596	p.kelbert	return Feature.OBSERVATION();
1490	fd1ce664	p.kelbert	// return getFeature("Specimen or observation");
1491	1de96f38	p.kelbert	}else{
1492			String message = "Unhandled record basis '%s' for defining individuals association feature type. Use default.";
1493			logger.warn(String.format(message, type.getMessage()));
1494	6490c596	p.kelbert	return Feature.OBSERVATION();
1495	fd1ce664	p.kelbert	// return getFeature("Specimen or observation");
1496	6490c596	p.kelbert
1497			}
1498			}
1499
1500	1d3f5c87	p.kelbert	/**
1501			* HandleIdentifications : get the scientific names present in the ABCD
1502			* document and store link them with the observation/specimen data
1503			* @param state: the current ABCD import state
1504			* @param derivedUnitFacade : the current derivedunitfacade
1505			*/
1506			private void handleIdentifications(Abcd206ImportState state, DerivedUnitFacade derivedUnitFacade) {
1507			Abcd206ImportConfigurator config = state.getConfig();
1508
1509
1510			String scientificName = "";
1511			boolean preferredFlag = false;
1512
1513	64260466	Patrick Plitzner	if (state.getDataHolder().nomenclatureCode == ""){
1514			state.getDataHolder().nomenclatureCode = config.getNomenclaturalCode().toString();
1515	1d3f5c87	p.kelbert	}
1516
1517	64260466	Patrick Plitzner	for (int i = 0; i < state.getDataHolder().identificationList.size(); i++) {
1518			Identification identification = state.getDataHolder().identificationList.get(i);
1519	8b77762f	Patric Plitzner	scientificName = identification.getScientificName().replaceAll(" et ", " & ");
1520	1d3f5c87	p.kelbert
1521	8b77762f	Patric Plitzner	String preferred = identification.getPreferred();
1522	64260466	Patrick Plitzner	if (preferred.equals("1") \|\| preferred.toLowerCase().indexOf("true") != -1 \|\| state.getDataHolder().identificationList.size()==1) {
1523	8b77762f	Patric Plitzner	preferredFlag = true;
1524	1d3f5c87	p.kelbert	}
1525			else {
1526	8b77762f	Patric Plitzner	preferredFlag = false;
1527	1d3f5c87	p.kelbert	}
1528	8b77762f	Patric Plitzner
1529			if (identification.getCode().indexOf(':') != -1) {
1530	64260466	Patrick Plitzner	state.getDataHolder().nomenclatureCode = identification.getCode().split(COLON)[1];
1531	1d3f5c87	p.kelbert	}
1532	8b77762f	Patric Plitzner	else{
1533	64260466	Patrick Plitzner	state.getDataHolder().nomenclatureCode = identification.getCode();
1534	1d3f5c87	p.kelbert	}
1535	8d425e15	Patric Plitzner	TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(scientificName, null, preferredFlag, state, i);
1536	ce8437b0	Patric Plitzner	Taxon taxon = getOrCreateTaxonForName(taxonName, state);
1537	1de96f38	p.kelbert	addTaxonNode(taxon, state,preferredFlag);
1538	1d3f5c87	p.kelbert	linkDeterminationEvent(state, taxon, preferredFlag, derivedUnitFacade);
1539			}
1540			}
1541
1542	18a5a690	Patric Plitzner
1543	8d425e15	Patric Plitzner	private TaxonNameBase<?, ?> getOrCreateTaxonName(String scientificName, Rank rank, boolean preferredFlag, Abcd206ImportState state, int unitIndexInAbcdFile){
1544	f2137d95	Patric Plitzner	TaxonNameBase<?, ?> taxonName = null;
1545	ce8437b0	Patric Plitzner	Abcd206ImportConfigurator config = state.getConfig();
1546	93186c1a	Patric Plitzner
1547			//check atomised name data for rank
1548			NonViralName<?> atomisedTaxonName = null;
1549	64260466	Patrick Plitzner	if (rank==null && unitIndexInAbcdFile>=0 && (state.getDataHolder().atomisedIdentificationList != null \|\| state.getDataHolder().atomisedIdentificationList.size() > 0)) {
1550			atomisedTaxonName = setTaxonNameByType(state.getDataHolder().atomisedIdentificationList.get(unitIndexInAbcdFile), scientificName, state);
1551	93186c1a	Patric Plitzner	if(atomisedTaxonName!=null){
1552			rank = atomisedTaxonName.getRank();
1553			}
1554			}
1555			if(rank==null){
1556	64260466	Patrick Plitzner	String message = "No rank specified for "+state.getDerivedUnitBase()+". Assuming species.";
1557	93186c1a	Patric Plitzner	report.addInfoMessage(message);
1558			logger.warn(message);
1559			rank = Rank.SPECIES();
1560			}
1561
1562	f5e6b489	Patric Plitzner	if(config.isReuseExistingTaxaWhenPossible()){
1563	64260466	Patrick Plitzner	NonViralName<?> parsedName = parseScientificName(scientificName, state);
1564	75efb034	Patric Plitzner	if(config.isIgnoreAuthorship() && parsedName!=null && preferredFlag){
1565	8d425e15	Patric Plitzner	// do not ignore authorship for non-preferred names because they need
1566			// to be created for the determination history
1567	75efb034	Patric Plitzner	String nameCache = parsedName.getNameCache();
1568			List<NonViralName> names = getNameService().findNamesByNameCache(nameCache, MatchMode.EXACT, null);
1569			taxonName = getBestMatchingName(scientificName, new ArrayList<TaxonNameBase>(names));
1570	f5e6b489	Patric Plitzner	}
1571			else {
1572			//search for existing names
1573			List<TaxonNameBase> names = getNameService().listByTitle(TaxonNameBase.class, scientificName, MatchMode.EXACT, null, null, null, null, null);
1574			taxonName = getBestMatchingName(scientificName, names);
1575	68d62a47	Patric Plitzner	}
1576	18a5a690	Patric Plitzner	}
1577	f2137d95	Patric Plitzner	else if (config.isParseNameAutomatically()){
1578	64260466	Patrick Plitzner	taxonName = parseScientificName(scientificName, state);
1579	93186c1a	Patric Plitzner	if(taxonName!=null){
1580			report.addName(taxonName);
1581			logger.info("Created new taxon name "+taxonName);
1582	ce8437b0	Patric Plitzner	}
1583			}
1584	93186c1a	Patric Plitzner	if(taxonName==null && atomisedTaxonName!=null){
1585			taxonName = atomisedTaxonName;
1586			report.addName(taxonName);
1587			logger.info("Created new taxon name "+taxonName);
1588			}
1589			else if(taxonName==null){
1590	ce8437b0	Patric Plitzner	//create new taxon name
1591			taxonName = NonViralName.NewInstance(rank);
1592			taxonName.setFullTitleCache(scientificName,true);
1593			taxonName.setTitleCache(scientificName, true);
1594	f2137d95	Patric Plitzner	report.addName(taxonName);
1595			logger.info("Created new taxon name "+taxonName);
1596	18a5a690	Patric Plitzner	}
1597			save(taxonName, state);
1598			return taxonName;
1599			}
1600
1601	ef183cff	Patric Plitzner	private TaxonNameBase<?, ?> getBestMatchingName(String scientificName, java.util.Collection<TaxonNameBase> names){
1602	f5e6b489	Patric Plitzner	List<TaxonNameBase> namesWithAcceptedTaxa = new ArrayList<TaxonNameBase>();
1603			for (TaxonNameBase name : names) {
1604			if(!name.getTaxa().isEmpty()){
1605			namesWithAcceptedTaxa.add(name);
1606			}
1607			}
1608			String message = "More than one taxon name was found for "+scientificName+"!";
1609			//check for names with accepted taxa
1610			if(namesWithAcceptedTaxa.size()>0){
1611			if(namesWithAcceptedTaxa.size()>1){
1612			report.addInfoMessage(message);
1613			logger.warn(message);
1614			return null;
1615			}
1616			return namesWithAcceptedTaxa.iterator().next();
1617			}
1618			//no names with accepted taxa found -> check accepted taxa of synonyms
1619			List<Taxon> taxaFromSynonyms = new ArrayList<Taxon>();
1620			for (TaxonNameBase name : names) {
1621			Set<TaxonBase> taxonBases = name.getTaxonBases();
1622			for (TaxonBase taxonBase : taxonBases) {
1623			if(taxonBase.isInstanceOf(Synonym.class)){
1624			Synonym synonym = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
1625			taxaFromSynonyms.addAll(synonym.getAcceptedTaxa());
1626	ef183cff	Patric Plitzner	}
1627			}
1628			}
1629	f5e6b489	Patric Plitzner	if(taxaFromSynonyms.size()>0){
1630			if(taxaFromSynonyms.size()>1){
1631			report.addInfoMessage(message);
1632			logger.warn(message);
1633			return null;
1634			}
1635			return taxaFromSynonyms.iterator().next().getName();
1636			}
1637	ef183cff	Patric Plitzner	return null;
1638			}
1639
1640	ce8437b0	Patric Plitzner	private Taxon getOrCreateTaxonForName(TaxonNameBase<?, ?> taxonNameBase, Abcd206ImportState state){
1641	18a5a690	Patric Plitzner	Set<Taxon> acceptedTaxa = taxonNameBase.getTaxa();
1642			if(acceptedTaxa.size()>0){
1643	ef183cff	Patric Plitzner	Taxon firstAcceptedTaxon = acceptedTaxa.iterator().next();
1644	18a5a690	Patric Plitzner	if(acceptedTaxa.size()>1){
1645	ef183cff	Patric Plitzner	String message = "More than one accepted taxon was found for taxon name: "
1646	64260466	Patrick Plitzner	+ taxonNameBase.getTitleCache() + "!\n" + firstAcceptedTaxon + "was chosen for "+state.getDerivedUnitBase();
1647	ef183cff	Patric Plitzner	report.addInfoMessage(message);
1648			logger.warn(message);
1649	18a5a690	Patric Plitzner	}
1650	75efb034	Patric Plitzner	else{
1651			return firstAcceptedTaxon;
1652			}
1653	18a5a690	Patric Plitzner	}
1654			else{
1655			Set<TaxonBase> taxonAndSynonyms = taxonNameBase.getTaxonBases();
1656	ce8437b0	Patric Plitzner	for (TaxonBase taxonBase : taxonAndSynonyms) {
1657			if(taxonBase.isInstanceOf(Synonym.class)){
1658			Synonym synonym = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
1659			Set<Taxon> acceptedTaxaOfSynonym = synonym.getAcceptedTaxa();
1660			if(acceptedTaxaOfSynonym.size()!=1){
1661	ef183cff	Patric Plitzner	String message = "No accepted taxa could be found for taxon name: "
1662			+ taxonNameBase.getTitleCache()
1663			+ "!\nEither it is a pro parte synonym or has no accepted taxa";
1664			report.addInfoMessage(message);
1665			logger.warn(message);
1666	ce8437b0	Patric Plitzner	}
1667			else{
1668	609a2f23	Patric Plitzner	return acceptedTaxaOfSynonym.iterator().next();
1669	ce8437b0	Patric Plitzner	}
1670	18a5a690	Patric Plitzner	}
1671			}
1672			}
1673	64260466	Patrick Plitzner	Taxon taxon = Taxon.NewInstance(taxonNameBase, state.getRef());
1674	ce8437b0	Patric Plitzner	save(taxon, state);
1675			report.addTaxon(taxon);
1676			logger.info("Created new taxon "+ taxon);
1677			return taxon;
1678	18a5a690	Patric Plitzner	}
1679
1680	1d3f5c87	p.kelbert	/**
1681			* @param taxon : a taxon to add as a node
1682			* @param state : the ABCD import state
1683			*/
1684	1de96f38	p.kelbert	private void addTaxonNode(Taxon taxon, Abcd206ImportState state, boolean preferredFlag) {
1685	1d3f5c87	p.kelbert	logger.info("link taxon to a taxonNode "+taxon.getTitleCache());
1686			boolean exist = false;
1687	64260466	Patrick Plitzner	Set<TaxonNode> allNodes = state.getClassification().getAllNodes();
1688	a8931728	Patric Plitzner	for (TaxonNode p : allNodes){
1689	1de96f38	p.kelbert	try{
1690			if(p.getTaxon().equals(taxon)) {
1691			exist =true;
1692			}
1693	8b77762f	Patric Plitzner	}
1694			catch(Exception e){
1695			logger.warn("TaxonNode doesn't seem to have a taxon");
1696			}
1697	1d3f5c87	p.kelbert	}
1698			if (!exist){
1699	22926545	Patric Plitzner	if(state.getConfig().isMoveNewTaxaToDefaultClassification()){
1700			addParentTaxon(taxon, state, preferredFlag, getDefaultClassification(state));
1701			}
1702			else{
1703	64260466	Patrick Plitzner	addParentTaxon(taxon, state, preferredFlag, state.getClassification());
1704	22926545	Patric Plitzner	}
1705	1d3f5c87	p.kelbert	}
1706			}
1707
1708	22926545	Patric Plitzner	private boolean hasTaxonNodeInClassification(Taxon taxon, Classification classification){
1709	06ca4e16	Patric Plitzner	if(taxon.getTaxonNodes()!=null){
1710			for (TaxonNode node : taxon.getTaxonNodes()){
1711			if(node.getClassification().equals(classification)){
1712			return true;
1713			}
1714	ff7173c1	Patric Plitzner	}
1715			}
1716			return false;
1717			}
1718
1719	1d3f5c87	p.kelbert	/**
1720			* Add the hierarchy for a Taxon(add higher taxa)
1721	22926545	Patric Plitzner	* @param classification
1722	1d3f5c87	p.kelbert	* @param taxon: a taxon to add as a node
1723			* @param state: the ABCD import state
1724			*/
1725	22926545	Patric Plitzner	private void addParentTaxon(Taxon taxon, Abcd206ImportState state, boolean preferredFlag, Classification classification){
1726	1d3f5c87	p.kelbert	NonViralName<?> nvname = CdmBase.deproxy(taxon.getName(), NonViralName.class);
1727			Rank rank = nvname.getRank();
1728			Taxon genus =null;
1729			Taxon subgenus =null;
1730			Taxon species = null;
1731			Taxon subspecies = null;
1732			Taxon parent = null;
1733			if (rank.isLower(Rank.GENUS() )){
1734			String prefix = nvname.getGenusOrUninomial();
1735	8d425e15	Patric Plitzner	TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(prefix, Rank.GENUS(), preferredFlag, state, -1);
1736	ce8437b0	Patric Plitzner	genus = getOrCreateTaxonForName(taxonName, state);
1737	1de96f38	p.kelbert	if (preferredFlag) {
1738	22926545	Patric Plitzner	parent = saveOrUpdateClassification(null, genus, classification, state);
1739	1de96f38	p.kelbert	}
1740	1d3f5c87	p.kelbert
1741			}
1742			if (rank.isLower(Rank.SUBGENUS())){
1743			String prefix = nvname.getGenusOrUninomial();
1744			String name = nvname.getInfraGenericEpithet();
1745			if (name != null){
1746	8d425e15	Patric Plitzner	TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(prefix+" "+name, Rank.SUBGENUS(), preferredFlag, state, -1);
1747	ce8437b0	Patric Plitzner	subgenus = getOrCreateTaxonForName(taxonName, state);
1748	1de96f38	p.kelbert	if (preferredFlag) {
1749	22926545	Patric Plitzner	parent = saveOrUpdateClassification(genus, subgenus, classification, state);
1750	1de96f38	p.kelbert	} }
1751	1d3f5c87	p.kelbert	}
1752			if (rank.isLower(Rank.SPECIES())){
1753			if (subgenus!=null){
1754			String prefix = nvname.getGenusOrUninomial();
1755			String name = nvname.getInfraGenericEpithet();
1756			String spe = nvname.getSpecificEpithet();
1757			if (spe != null){
1758	8d425e15	Patric Plitzner	TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(prefix+" "+name+" "+spe, Rank.SPECIES(), preferredFlag, state, -1);
1759	ce8437b0	Patric Plitzner	species = getOrCreateTaxonForName(taxonName, state);
1760	1de96f38	p.kelbert	if (preferredFlag) {
1761	22926545	Patric Plitzner	parent = saveOrUpdateClassification(subgenus, species, classification, state);
1762	1de96f38	p.kelbert	}
1763	1d3f5c87	p.kelbert	}
1764			}
1765			else{
1766			String prefix = nvname.getGenusOrUninomial();
1767			String name = nvname.getSpecificEpithet();
1768			if (name != null){
1769	8d425e15	Patric Plitzner	TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(prefix+" "+name, Rank.SPECIES(), preferredFlag, state, -1);
1770	ce8437b0	Patric Plitzner	species = getOrCreateTaxonForName(taxonName, state);
1771	1de96f38	p.kelbert	if (preferredFlag) {
1772	22926545	Patric Plitzner	parent = saveOrUpdateClassification(genus, species, classification, state);
1773	1de96f38	p.kelbert	}
1774	1d3f5c87	p.kelbert	}
1775			}
1776			}
1777	18a5a690	Patric Plitzner	if (rank.isLower(Rank.INFRASPECIES())){
1778	8d425e15	Patric Plitzner	TaxonNameBase<?,?> taxonName = getOrCreateTaxonName(nvname.getFullTitleCache(), Rank.SUBSPECIES(), preferredFlag, state, -1);
1779	ce8437b0	Patric Plitzner	subspecies = getOrCreateTaxonForName(taxonName, state);
1780	1de96f38	p.kelbert	if (preferredFlag) {
1781	22926545	Patric Plitzner	parent = saveOrUpdateClassification(species, subspecies, classification, state);
1782	1de96f38	p.kelbert	}
1783			}
1784	71662a8c	Patric Plitzner	if (preferredFlag && parent!=taxon) {
1785	22926545	Patric Plitzner	saveOrUpdateClassification(parent, taxon, classification, state);
1786	1d3f5c87	p.kelbert	}
1787			}
1788
1789			/**
1790			* Link a parent to a child and save it in the current classification
1791			* @param parent: the higher Taxon
1792			* @param child : the lower (or current) Taxon
1793			* return the Taxon from the new created Node
1794	22926545	Patric Plitzner	* @param classification
1795	1de96f38	p.kelbert	* @param state
1796	1d3f5c87	p.kelbert	*/
1797	22926545	Patric Plitzner	private Taxon saveOrUpdateClassification(Taxon parent, Taxon child, Classification classification, Abcd206ImportState state) {
1798	1de96f38	p.kelbert	TaxonNode node =null;
1799			if (parent != null) {
1800			parent = (Taxon) getTaxonService().find(parent.getUuid());
1801			child = (Taxon) getTaxonService().find(child.getUuid());
1802	ff7173c1	Patric Plitzner	//here we do not have to check if the taxon nodes already exists
1803			//this is done by classification.addParentChild()
1804	b4426234	Patric Plitzner	//do not add child node if it already exists
1805	22926545	Patric Plitzner	if(hasTaxonNodeInClassification(child, classification)){
1806	b4426234	Patric Plitzner	return child;
1807			}
1808	8d425e15	Patric Plitzner	else{
1809	64260466	Patrick Plitzner	node = classification.addParentChild(parent, child, state.getRef(), "");
1810	22926545	Patric Plitzner	save(classification, state);
1811	8d425e15	Patric Plitzner	}
1812	1de96f38	p.kelbert	}
1813	ff7173c1	Patric Plitzner	else {
1814	1de96f38	p.kelbert	child = (Taxon) getTaxonService().find(child.getUuid());
1815	ff7173c1	Patric Plitzner	//do not add child node if it already exists
1816	22926545	Patric Plitzner	if(hasTaxonNodeInClassification(child, classification)){
1817	ff7173c1	Patric Plitzner	return child;
1818			}
1819	8d425e15	Patric Plitzner	else{
1820	64260466	Patrick Plitzner	node = classification.addChildTaxon(child, state.getRef(), null);
1821	22926545	Patric Plitzner	save(classification, state);
1822	8d425e15	Patric Plitzner	}
1823	1de96f38	p.kelbert	}
1824	71662a8c	Patric Plitzner	if(node!=null){
1825			report.addTaxonNode(node);
1826			return node.getTaxon();
1827			}
1828			String message = "Could not create taxon node for " +child;
1829			report.addInfoMessage(message);
1830			logger.warn(message);
1831			return null;
1832	1d3f5c87	p.kelbert	}
1833	1de96f38	p.kelbert
1834	1d3f5c87	p.kelbert	/**
1835			* Parse automatically the scientific name
1836	64260466	Patrick Plitzner	* @param state
1837	1d3f5c87	p.kelbert	* @param scientificName: the scientific name to parse
1838			* @return a parsed name
1839			*/
1840	64260466	Patrick Plitzner	private NonViralName<?> parseScientificName(String scientificName, Abcd206ImportState state) {
1841	1d3f5c87	p.kelbert	NonViralNameParserImpl nvnpi = NonViralNameParserImpl.NewInstance();
1842			NonViralName<?> taxonName = null;
1843			boolean problem = false;
1844
1845			if(DEBUG){
1846	64260466	Patrick Plitzner	logger.info("parseScientificName " + state.getDataHolder().nomenclatureCode.toString());
1847	1d3f5c87	p.kelbert	}
1848
1849	64260466	Patrick Plitzner	if (state.getDataHolder().nomenclatureCode.toString().equals("Zoological") \|\| state.getDataHolder().nomenclatureCode.toString().contains("ICZN")) {
1850	1d3f5c87	p.kelbert	taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICZN, null);
1851			if (taxonName.hasProblem()) {
1852			problem = true;
1853			}
1854			}
1855	64260466	Patrick Plitzner	if (state.getDataHolder().nomenclatureCode.toString().equals("Botanical") \|\| state.getDataHolder().nomenclatureCode.toString().contains("ICBN")) {
1856	59c2c696	Andreas Müller	taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNAFP, null);
1857	1d3f5c87	p.kelbert	if (taxonName.hasProblem()) {
1858			problem = true;
1859			}
1860			}
1861	64260466	Patrick Plitzner	if (state.getDataHolder().nomenclatureCode.toString().equals("Bacterial") \|\| state.getDataHolder().nomenclatureCode.toString().contains("ICBN")) {
1862	1d3f5c87	p.kelbert	taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNB, null);
1863			if (taxonName.hasProblem()) {
1864			problem = true;
1865			}
1866			}
1867	64260466	Patrick Plitzner	if (state.getDataHolder().nomenclatureCode.toString().equals("Cultivar") \|\| state.getDataHolder().nomenclatureCode.toString().contains("ICNCP")) {
1868	1d3f5c87	p.kelbert	taxonName = nvnpi.parseFullName(scientificName, NomenclaturalCode.ICNCP, null);
1869			if (taxonName.hasProblem()) {
1870			problem = true;
1871			}
1872			}
1873			if (problem) {
1874			logger.info("Parsing with problem in parseScientificName " + scientificName);
1875			return null;
1876			}
1877			return taxonName;
1878
1879			}
1880
1881			/**
1882			* Create the name without automatic parsing, either because it failed, or because the user deactivated it.
1883			* The name is built upon the ABCD fields
1884			* @param atomisedMap : the ABCD atomised fields
1885			* @param fullName : the full scientific name
1886	64260466	Patrick Plitzner	* @param state
1887	1d3f5c87	p.kelbert	* @return the corresponding Botanical or Zoological or... name
1888			*/
1889			private NonViralName<?> setTaxonNameByType(
1890	64260466	Patrick Plitzner	HashMap<String, String> atomisedMap, String fullName, Abcd206ImportState state) {
1891	1d3f5c87	p.kelbert	boolean problem = false;
1892			if(DEBUG) {
1893	64260466	Patrick Plitzner	logger.info("settaxonnamebytype " + state.getDataHolder().nomenclatureCode.toString());
1894	1d3f5c87	p.kelbert	}
1895
1896	64260466	Patrick Plitzner	if (state.getDataHolder().nomenclatureCode.equals("Zoological")) {
1897	1d3f5c87	p.kelbert	NonViralName<ZoologicalName> taxonName = ZoologicalName.NewInstance(null);
1898			taxonName.setFullTitleCache(fullName, true);
1899			taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
1900			taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
1901			taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap,"SpeciesEpithet")));
1902			taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap,"SubspeciesEpithet")));
1903
1904			if (taxonName.getGenusOrUninomial() != null){
1905			taxonName.setRank(Rank.GENUS());
1906			}
1907
1908	1de96f38	p.kelbert	if (taxonName.getInfraGenericEpithet() != null){
1909	1d3f5c87	p.kelbert	taxonName.setRank(Rank.SUBGENUS());
1910			}
1911
1912	1de96f38	p.kelbert	if (taxonName.getSpecificEpithet() != null){
1913	1d3f5c87	p.kelbert	taxonName.setRank(Rank.SPECIES());
1914			}
1915
1916	1de96f38	p.kelbert	if (taxonName.getInfraSpecificEpithet() != null){
1917	1d3f5c87	p.kelbert	taxonName.setRank(Rank.SUBSPECIES());
1918			}
1919
1920			Team team = null;
1921			if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
1922			team = Team.NewInstance();
1923			team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
1924			}
1925			else {
1926			if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
1927			team = Team.NewInstance();
1928			team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
1929			}
1930			}
1931			if (team != null) {
1932	ce7c7aa3	Andreas Müller	taxonName.setBasionymAuthorship(team);
1933	1d3f5c87	p.kelbert	}
1934			else {
1935			if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
1936			taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
1937			}
1938			else if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
1939			taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamAndYear"));
1940			}
1941			}
1942			if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
1943			team = Team.NewInstance();
1944			team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
1945	ce7c7aa3	Andreas Müller	taxonName.setCombinationAuthorship(team);
1946	1d3f5c87	p.kelbert	}
1947			if (taxonName.hasProblem()) {
1948			logger.info("pb ICZN");
1949			problem = true;
1950			}
1951			else {
1952			return taxonName;
1953			}
1954			}
1955	64260466	Patrick Plitzner	else if (state.getDataHolder().nomenclatureCode.equals("Botanical")) {
1956			BotanicalName taxonName = (BotanicalName) parseScientificName(fullName, state);
1957	1d3f5c87	p.kelbert	if (taxonName != null){
1958			return taxonName;
1959			}
1960			else{
1961			taxonName = BotanicalName.NewInstance(null);
1962			}
1963			taxonName.setFullTitleCache(fullName, true);
1964			taxonName.setGenusOrUninomial(NB(getFromMap(atomisedMap, "Genus")));
1965			taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "FirstEpithet")));
1966			taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "InfraSpeEpithet")));
1967			try {
1968			taxonName.setRank(Rank.getRankByName(getFromMap(atomisedMap, "Rank")));
1969			} catch (Exception e) {
1970			if (taxonName.getGenusOrUninomial() != null){
1971			taxonName.setRank(Rank.GENUS());
1972			}
1973			else if (taxonName.getInfraGenericEpithet() != null){
1974			taxonName.setRank(Rank.SUBGENUS());
1975			}
1976			else if (taxonName.getSpecificEpithet() != null){
1977			taxonName.setRank(Rank.SPECIES());
1978			}
1979			else if (taxonName.getInfraSpecificEpithet() != null){
1980			taxonName.setRank(Rank.SUBSPECIES());
1981			}
1982			}
1983			Team team = null;
1984			if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
1985			team = Team.NewInstance();
1986			team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"), true);
1987	ce7c7aa3	Andreas Müller	taxonName.setBasionymAuthorship(team);
1988	1d3f5c87	p.kelbert	}
1989			if (getFromMap(atomisedMap, "AuthorTeam") != null) {
1990			team = Team.NewInstance();
1991			team.setTitleCache(getFromMap(atomisedMap, "AuthorTeam"), true);
1992	ce7c7aa3	Andreas Müller	taxonName.setCombinationAuthorship(team);
1993	1d3f5c87	p.kelbert	}
1994			if (team == null) {
1995			if (getFromMap(atomisedMap, "AuthorTeamParenthesis") != null) {
1996			taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeamParenthesis"));
1997			}
1998			else if (getFromMap(atomisedMap, "AuthorTeam") != null) {
1999			taxonName.setAuthorshipCache(getFromMap(atomisedMap, "AuthorTeam"));
2000			}
2001			}
2002			if (getFromMap(atomisedMap, "CombinationAuthorTeamAndYear") != null) {
2003			team = Team.NewInstance();
2004			team.setTitleCache(getFromMap(atomisedMap, "CombinationAuthorTeamAndYear"), true);
2005	ce7c7aa3	Andreas Müller	taxonName.setCombinationAuthorship(team);
2006	1d3f5c87	p.kelbert	}
2007			if (taxonName.hasProblem()) {
2008			logger.info("pb ICBN");
2009			problem = true;
2010			}
2011			else {
2012			return taxonName;
2013			}
2014			}
2015	64260466	Patrick Plitzner	else if (state.getDataHolder().nomenclatureCode.equals("Bacterial")) {
2016	1d3f5c87	p.kelbert	NonViralName<BacterialName> taxonName = BacterialName.NewInstance(null);
2017			taxonName.setFullTitleCache(fullName, true);
2018			taxonName.setGenusOrUninomial(getFromMap(atomisedMap, "Genus"));
2019			taxonName.setInfraGenericEpithet(NB(getFromMap(atomisedMap, "SubGenus")));
2020			taxonName.setSpecificEpithet(NB(getFromMap(atomisedMap, "Species")));
2021			taxonName.setInfraSpecificEpithet(NB(getFromMap(atomisedMap, "SubspeciesEpithet")));
2022
2023			if (taxonName.getGenusOrUninomial() != null){
2024			taxonName.setRank(Rank.GENUS());
2025			}
2026			else if (taxonName.getInfraGenericEpithet() != null){
2027			taxonName.setRank(Rank.SUBGENUS());
2028			}
2029			else if (taxonName.getSpecificEpithet() != null){
2030			taxonName.setRank(Rank.SPECIES());
2031			}
2032			else if (taxonName.getInfraSpecificEpithet() != null){
2033			taxonName.setRank(Rank.SUBSPECIES());
2034			}
2035
2036			if (getFromMap(atomisedMap, "AuthorTeamAndYear") != null) {
2037			Team team = Team.NewInstance();
2038			team.setTitleCache(getFromMap(atomisedMap, "AuthorTeamAndYear"), true);
2039	ce7c7aa3	Andreas Müller	taxonName.setCombinationAuthorship(team);
2040	1d3f5c87	p.kelbert	}
2041			if (getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear") != null) {
2042			Team team = Team.NewInstance();
2043			team.setTitleCache(getFromMap(atomisedMap, "ParentheticalAuthorTeamAndYear"), true);
2044	ce7c7aa3	Andreas Müller	taxonName.setBasionymAuthorship(team);
2045	1d3f5c87	p.kelbert	}
2046			if (taxonName.hasProblem()) {
2047			logger.info("pb ICNB");
2048			problem = true;
2049			}
2050			else {
2051			return taxonName;
2052			}
2053			}
2054	64260466	Patrick Plitzner	else if (state.getDataHolder().nomenclatureCode.equals("Cultivar")) {
2055	1d3f5c87	p.kelbert	CultivarPlantName taxonName = CultivarPlantName.NewInstance(null);
2056
2057			if (taxonName.hasProblem()) {
2058			logger.info("pb ICNCP");
2059			problem = true;
2060			}
2061			else {
2062			return taxonName;
2063			}
2064			return taxonName;
2065			}
2066
2067			if (problem) {
2068			logger.info("Problem im setTaxonNameByType ");
2069			NonViralName<?> taxonName = NonViralName.NewInstance(null);
2070			taxonName.setFullTitleCache(fullName, true);
2071			return taxonName;
2072			}
2073			NonViralName<?> tn = NonViralName.NewInstance(null);
2074			return tn;
2075			}
2076
2077
2078			/**
2079			* Get a formated string from a hashmap
2080			* @param atomisedMap
2081			* @param key
2082			* @return
2083			*/
2084			private String getFromMap(HashMap<String, String> atomisedMap, String key) {
2085			String value = null;
2086			if (atomisedMap.containsKey(key)) {
2087			value = atomisedMap.get(key);
2088			}
2089
2090			try {
2091			if (value != null && key.matches(".Year.")) {
2092			value = value.trim();
2093			if (value.matches("[a-z A-Z ]*[0-9]{4}$")) {
2094			String tmp = value.split("[0-9]{4}$")[0];
2095			int year = Integer.parseInt(value.split(tmp)[1]);
2096			if (year >= 1752) {
2097			value = tmp;
2098			}
2099			else {
2100			value = null;
2101			}
2102			}
2103			else {
2104			value = null;
2105			}
2106			}
2107			}
2108			catch (Exception e) {
2109			value = null;
2110			}
2111			return value;
2112			}
2113
2114			// private void compareABCDtoCDM(URI urlFileName, List<String> knownElts, Abcd206XMLFieldGetter abcdFieldGetter) {
2115			// try {
2116			// DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
2117			// DocumentBuilder constructeur = factory.newDocumentBuilder();
2118			// URL url = urlFileName.toURL();
2119			// Object o = url.getContent();
2120			// InputStream is = (InputStream) o;
2121			// Document document = constructeur.parse(is);
2122			// Element root = document.getDocumentElement();
2123			// abcdFieldGetter.traverse(root);
2124			// }
2125			// catch (ParserConfigurationException e){
2126			// e.printStackTrace();
2127			// }
2128			// catch (SAXException e) {
2129			// e.printStackTrace();
2130			// }
2131			// catch (IOException e) {
2132			// e.printStackTrace();
2133			// }
2134	64260466	Patrick Plitzner	// Set<String> elts = state.getDataHolder().allABCDelements.keySet();
2135	1d3f5c87	p.kelbert	// Iterator<String> it = elts.iterator();
2136			// String elt;
2137			// while (it.hasNext()) {
2138			// elt = it.next();
2139			// if (knownElts.indexOf(elt) == -1) {
2140			// if(DEBUG) {
2141	64260466	Patrick Plitzner	// logger.info("Unmerged ABCD element: " + elt + " - "+ state.getDataHolder().allABCDelements.get(elt));
2142	1d3f5c87	p.kelbert	// }
2143			// }
2144			// }
2145			// }
2146
2147			/**
2148			* Load the list of names from the ABCD file and save them
2149			* @param state : the current ABCD import state
2150			* @param unitsList : the unit list from the ABCD file
2151			* @param abcdFieldGetter : the ABCD parser
2152			*/
2153			private void prepareCollectors(Abcd206ImportState state, NodeList unitsList, Abcd206XMLFieldGetter abcdFieldGetter) {
2154			List<String> collectors = new ArrayList<String>();
2155			List<String> teams = new ArrayList<String>();
2156			List<List<String>> collectorinteams = new ArrayList<List<String>>();
2157
2158			for (int i = 0; i < unitsList.getLength(); i++) {
2159	64260466	Patrick Plitzner	this.getCollectorsFromXML((Element) unitsList.item(i), abcdFieldGetter, state);
2160			for (String agent : state.getDataHolder().gatheringAgentList) {
2161	1d3f5c87	p.kelbert	collectors.add(agent);
2162			}
2163	64260466	Patrick Plitzner	List<String> tmpTeam = new ArrayList<String>(new HashSet<String>(state.getDataHolder().gatheringTeamList));
2164	1d3f5c87	p.kelbert	if(!tmpTeam.isEmpty()) {
2165			teams.add(StringUtils.join(tmpTeam.toArray()," & "));
2166			}
2167			for (String agent:tmpTeam) {
2168			collectors.add(agent);
2169			}
2170			}
2171
2172			List<String> collectorsU = new ArrayList<String>(new HashSet<String>(collectors));
2173			List<String> teamsU = new ArrayList<String>(new HashSet<String>(teams));
2174
2175
2176			//existing teams in DB
2177			Map<String,Team> titleCacheTeam = new HashMap<String, Team>();
2178			List<UuidAndTitleCache<Team>> hiberTeam = getAgentService().getTeamUuidAndTitleCache();
2179
2180			Set<UUID> uuids = new HashSet<UUID>();
2181			for (UuidAndTitleCache<Team> hibernateT:hiberTeam){
2182			uuids.add(hibernateT.getUuid());
2183			}
2184			if (!uuids.isEmpty()){
2185			List<AgentBase> existingTeams = getAgentService().find(uuids);
2186			for (AgentBase<?> existingP:existingTeams){
2187			titleCacheTeam.put(existingP.getTitleCache(),CdmBase.deproxy(existingP,Team.class));
2188			}
2189			}
2190
2191
2192			Map<String,UUID> teamMap = new HashMap<String, UUID>();
2193			for (UuidAndTitleCache<Team> uuidt:hiberTeam){
2194			teamMap.put(uuidt.getTitleCache(), uuidt.getUuid());
2195			}
2196
2197			//existing persons in DB
2198			List<UuidAndTitleCache<Person>> hiberPersons = getAgentService().getPersonUuidAndTitleCache();
2199			Map<String,Person> titleCachePerson = new HashMap<String, Person>();
2200			uuids = new HashSet<UUID>();
2201			for (UuidAndTitleCache<Person> hibernateP:hiberPersons){
2202			uuids.add(hibernateP.getUuid());
2203			}
2204
2205			if (!uuids.isEmpty()){
2206			List<AgentBase> existingPersons = getAgentService().find(uuids);
2207			for (AgentBase<?> existingP:existingPersons){
2208			titleCachePerson.put(existingP.getTitleCache(),CdmBase.deproxy(existingP,Person.class));
2209			}
2210			}
2211
2212			Map<String,UUID> personMap = new HashMap<String, UUID>();
2213			for (UuidAndTitleCache<Person> person:hiberPersons){
2214			personMap.put(person.getTitleCache(), person.getUuid());
2215			}
2216
2217	f96df645	Andreas Müller	java.util.Collection<Person> personToadd = new ArrayList<Person>();
2218			java.util.Collection<Team> teamToAdd = new ArrayList<Team>();
2219	1d3f5c87	p.kelbert
2220			for (String collector:collectorsU){
2221			Person p = Person.NewInstance();
2222			p.setTitleCache(collector,true);
2223			if (!personMap.containsKey(p.getTitleCache())){
2224			personToadd.add(p);
2225			}
2226			}
2227			for (String team:teamsU){
2228			Team p = Team.NewInstance();
2229			p.setTitleCache(team,true);
2230			if (!teamMap.containsKey(p.getTitleCache())){
2231			teamToAdd.add(p);
2232			}
2233			}
2234
2235			if(!personToadd.isEmpty()){
2236	f96df645	Andreas Müller	for (Person agent: personToadd){
2237	1de96f38	p.kelbert	save(agent, state);
2238	f96df645	Andreas Müller	titleCachePerson.put(agent.getTitleCache(),CdmBase.deproxy(agent, Person.class) );
2239	1d3f5c87	p.kelbert	}
2240			}
2241
2242			Person ptmp ;
2243			Map <String,Integer>teamdone = new HashMap<String, Integer>();
2244			for (List<String> collteam: collectorinteams){
2245			if (!teamdone.containsKey(StringUtils.join(collteam.toArray(),"-"))){
2246			Team team = new Team();
2247			boolean em =true;
2248			for (String collector:collteam){
2249			ptmp = Person.NewInstance();
2250			ptmp.setTitleCache(collector,true);
2251			Person p2 = titleCachePerson.get(ptmp.getTitleCache());
2252			team.addTeamMember(p2);
2253			em=false;
2254			}
2255			if (!em) {
2256			teamToAdd.add(team);
2257			}
2258			teamdone.put(StringUtils.join(collteam.toArray(),"-"),0);
2259			}
2260			}
2261
2262			if(!teamToAdd.isEmpty()){
2263	1de96f38	p.kelbert	for (Team agent: teamToAdd){
2264			save(agent, state);
2265	f96df645	Andreas Müller	titleCacheTeam.put(agent.getTitleCache(), CdmBase.deproxy( agent,Team.class) );
2266	1d3f5c87	p.kelbert	}
2267			}
2268
2269			state.getConfig().setTeams(titleCacheTeam);
2270			state.getConfig().setPersons(titleCachePerson);
2271			}
2272
2273			@Override
2274			protected boolean isIgnore(Abcd206ImportState state) {
2275			return false;
2276			}
2277	636583bf	Andreas Müller
2278	ef183cff	Patric Plitzner	/**
2279			* Gets the default classification. If <code>null</code> creates it
2280			* @param state the current import state
2281			* @return the default classification
2282			*/
2283			public Classification getDefaultClassification(Abcd206ImportState state) {
2284	64260466	Patrick Plitzner	if(state.getDefaultClassification()==null){
2285	e2e537ff	Patric Plitzner	List<Classification> classificationList = getClassificationService().list(Classification.class, null, null, null, null);
2286			for (Classification classif : classificationList){
2287	64260466	Patrick Plitzner	if (classif.getTitleCache().equalsIgnoreCase(DEFAULT_CLASSIFICATION_ABCD) && classif.getCitation().equals(state.getRef())) {
2288			state.setDefaultClassification(classif);
2289	e2e537ff	Patric Plitzner	break;
2290			}
2291			}
2292	64260466	Patrick Plitzner	state.setDefaultClassification(Classification.NewInstance(DEFAULT_CLASSIFICATION_ABCD, state.getRef(), Language.DEFAULT()));
2293			save(state.getDefaultClassification(), state);
2294	ef183cff	Patric Plitzner	}
2295	64260466	Patrick Plitzner	return state.getDefaultClassification();
2296	ef183cff	Patric Plitzner	}
2297
2298	5131af90	Patric Plitzner	}

Project

General

Profile

EDIT