Project

General

Profile

Download (36.1 KB) Statistics
| Branch: | Revision:
1
/**
2
* Copyright (C) 2007 EDIT
3
* European Distributed Institute of Taxonomy
4
* http://www.e-taxonomy.eu
5
*
6
* The contents of this file are subject to the Mozilla Public License Version 1.1
7
* See LICENSE.TXT at the top of this package for the full license terms.
8
*/
9

    
10
package eu.etaxonomy.cdm.io.berlinModel.in;
11

    
12
import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.TAX_REL_IS_HETEROTYPIC_SYNONYM_OF;
13
import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.TAX_REL_IS_HOMOTYPIC_SYNONYM_OF;
14
import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.TAX_REL_IS_INCLUDED_IN;
15
import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.TAX_REL_IS_MISAPPLIED_NAME_OF;
16
import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.TAX_REL_IS_PARTIAL_HETEROTYPIC_SYNONYM_OF;
17
import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.TAX_REL_IS_PARTIAL_HOMOTYPIC_SYNONYM_OF;
18
import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.TAX_REL_IS_PARTIAL_SYN_OF;
19
import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.TAX_REL_IS_PROPARTE_HETEROTYPIC_SYNONYM_OF;
20
import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.TAX_REL_IS_PROPARTE_HOMOTYPIC_SYNONYM_OF;
21
import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.TAX_REL_IS_PROPARTE_SYN_OF;
22
import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.TAX_REL_IS_SYNONYM_OF;
23

    
24
import java.sql.ResultSet;
25
import java.sql.SQLException;
26
import java.util.HashMap;
27
import java.util.HashSet;
28
import java.util.Map;
29
import java.util.Set;
30
import java.util.UUID;
31

    
32
import org.apache.commons.lang.StringUtils;
33
import org.apache.log4j.Logger;
34
import org.springframework.stereotype.Component;
35
import org.springframework.transaction.TransactionStatus;
36

    
37
import eu.etaxonomy.cdm.common.CdmUtils;
38
import eu.etaxonomy.cdm.common.ResultWrapper;
39
import eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer;
40
import eu.etaxonomy.cdm.io.berlinModel.in.validation.BerlinModelTaxonRelationImportValidator;
41
import eu.etaxonomy.cdm.io.common.IOValidator;
42
import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
43
import eu.etaxonomy.cdm.io.common.Source;
44
import eu.etaxonomy.cdm.model.common.AnnotatableEntity;
45
import eu.etaxonomy.cdm.model.common.CdmBase;
46
import eu.etaxonomy.cdm.model.common.IdentifiableSource;
47
import eu.etaxonomy.cdm.model.name.NameRelationshipType;
48
import eu.etaxonomy.cdm.model.name.NomenclaturalStatus;
49
import eu.etaxonomy.cdm.model.name.NomenclaturalStatusType;
50
import eu.etaxonomy.cdm.model.name.TaxonName;
51
import eu.etaxonomy.cdm.model.reference.Reference;
52
import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
53
import eu.etaxonomy.cdm.model.taxon.Classification;
54
import eu.etaxonomy.cdm.model.taxon.Synonym;
55
import eu.etaxonomy.cdm.model.taxon.SynonymType;
56
import eu.etaxonomy.cdm.model.taxon.Taxon;
57
import eu.etaxonomy.cdm.model.taxon.TaxonBase;
58
import eu.etaxonomy.cdm.model.taxon.TaxonNode;
59
import eu.etaxonomy.cdm.model.taxon.TaxonRelationship;
60
import eu.etaxonomy.cdm.model.taxon.TaxonRelationshipType;
61
import eu.etaxonomy.cdm.strategy.exceptions.UnknownCdmTypeException;
62
import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
63
import eu.etaxonomy.cdm.strategy.parser.TimePeriodParser;
64

    
65
/**
66
 * @author a.mueller
67
 * @since 20.03.2008
68
 */
69
@Component
70
public class BerlinModelTaxonRelationImport  extends BerlinModelImportBase  {
71
    private static final long serialVersionUID = -7234926279240842557L;
72

    
73
    private static final Logger logger = Logger.getLogger(BerlinModelTaxonRelationImport.class);
74

    
75
	public static final String TREE_NAMESPACE = "PTRefFk";
76
	private static final Integer AUCT_REF_ID = 5959;
77

    
78
	private static int modCount = 30000;
79
	private static final String pluralString = "taxon relations";
80
	private static final String dbTableName = "RelPTaxon";
81

    
82

    
83
	public BerlinModelTaxonRelationImport(){
84
		super(dbTableName, pluralString);
85
	}
86

    
87
	/**
88
	 * Creates a classification for each PTaxon reference which belongs to a taxon that is
89
	 * included at least in one <i>taxonomically included</i> relationship.
90
	 *
91
	 * @param state
92
	 * @throws SQLException
93
	 */
94
	private void makeClassifications(BerlinModelImportState state) throws SQLException{
95
		logger.info("start make classification ...");
96

    
97
		Set<String> idSet = getTreeReferenceIdSet(state);
98

    
99
		//reference map
100
		String nameSpace = BerlinModelReferenceImport.REFERENCE_NAMESPACE;
101
		Class<?> cdmClass = Reference.class;
102
		@SuppressWarnings("unchecked")
103
        Map<String, Reference> refMap = (Map<String, Reference>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
104

    
105
		String classificationName = "Classification - No Name";
106

    
107
		ResultSet rs = state.getConfig().getSource().getResultSet(getClassificationQuery(state)) ;
108
		int i = 0;
109
		//for each reference
110
		try {
111
			//TODO handle case useSingleClassification = true && sourceSecId = null, which returns no record
112
			boolean isFirst = true;
113
		    while (rs.next()){
114

    
115
				try {
116
					if ((i++ % modCount) == 0 && i!= 1 ){ logger.info("RelPTaxa handled: " + (i-1));}
117

    
118
					Integer ptRefFkInt = nullSafeInt(rs,"PTRefFk");
119
					String ptRefFk= String.valueOf(ptRefFkInt);
120
					Reference ref = refMap.get(ptRefFk);
121

    
122
					String refCache = rs.getString("RefCache");
123
					if (isNotBlank(refCache)){
124
						classificationName = refCache;
125
					}
126
					if (ref != null && StringUtils.isNotBlank(ref.getTitleCache())){
127
						classificationName = ref.getTitleCache();
128
					}
129
					if (isFirst && isNotBlank(state.getConfig().getClassificationName())){
130
					    classificationName = state.getConfig().getClassificationName();
131
					}
132
					Classification tree = Classification.NewInstance(classificationName);
133
					tree.setReference(ref);
134
					if (i == 1 && state.getConfig().getClassificationUuid() != null){
135
						tree.setUuid(state.getConfig().getClassificationUuid());
136
					}
137
					IdentifiableSource identifiableSource = IdentifiableSource.NewDataImportInstance(ptRefFk, TREE_NAMESPACE);
138
					tree.addSource(identifiableSource);
139

    
140
					getClassificationService().save(tree);
141
					state.putClassificationUuidInt(ptRefFkInt, tree);
142
					isFirst = false;
143
				} catch (Exception e) {
144
					logger.error("Error in BerlinModleTaxonRelationImport.makeClassifications: " + e.getMessage());
145
					e.printStackTrace();
146
				}
147
			}
148
		} catch (SQLException e) {
149
			logger.error("Error in BerlinModleTaxonRelationImport.makeClassifications: " + e.getMessage());
150
			throw e;
151
		}
152
		logger.info("end make classification ...");
153

    
154
		return;
155
	}
156

    
157
	/**
158
	 * @return
159
	 * @throws SQLException
160
	 */
161
	private Set<String> getTreeReferenceIdSet(BerlinModelImportState state) throws SQLException {
162
		Source source = state.getConfig().getSource();
163
		Set<String> result = new HashSet<String>();
164
		ResultSet rs = source.getResultSet(getClassificationQuery(state)) ;
165
		while (rs.next()){
166
			Object id = rs.getObject("PTRefFk");
167
			result.add(String.valueOf(id));
168
		}
169
		return result;
170
	}
171

    
172
	private String getClassificationQuery(BerlinModelImportState state) {
173
		boolean includeAllClassifications = state.getConfig().isIncludeAllNonMisappliedRelatedClassifications();
174
		String strQuerySelect = "SELECT PTaxon.PTRefFk, r.RefCache ";
175
		String strQueryFrom = " FROM RelPTaxon " +
176
							" INNER JOIN PTaxon AS PTaxon ON RelPTaxon.PTNameFk2 = PTaxon.PTNameFk AND RelPTaxon.PTRefFk2 = PTaxon.PTRefFk " +
177
							" INNER JOIN Reference r ON PTaxon.PTRefFk = r.RefId ";
178
		String strQueryWhere = " WHERE (RelPTaxon.RelQualifierFk = 1) ";
179
		if (includeAllClassifications){
180
			strQueryWhere = " WHERE (RelPTaxon.RelQualifierFk <> 3) ";
181
		}else{
182
			if (state.getConfig().isUseSingleClassification()){
183
				if (state.getConfig().getSourceSecId()!= null){
184
					strQueryWhere += " AND PTaxon.PTRefFk = " + state.getConfig().getSourceSecId() +  " ";
185
				}else{
186
					strQueryWhere += " AND (1=0) ";
187
				}
188
			}
189
		}
190

    
191
		String strQueryGroupBy = " GROUP BY PTaxon.PTRefFk, r.RefCache ";
192
		String strQuery = strQuerySelect + " " + strQueryFrom + " " + strQueryWhere + " " + strQueryGroupBy;
193

    
194

    
195
		if (includeAllClassifications){
196
			//add otherdirection
197
			strQuerySelect = "SELECT PTaxon.PTRefFk, r.RefCache ";
198
			strQueryFrom = " FROM RelPTaxon rel " +
199
								" INNER JOIN PTaxon AS PTaxon ON rel.PTNameFk1 = PTaxon.PTNameFk AND rel.PTRefFk1 = PTaxon.PTRefFk " +
200
								" INNER JOIN Reference r ON PTaxon.PTRefFk = r.RefId ";
201
			strQueryWhere =" WHERE (rel.RelQualifierFk <> 3) ";
202
			String strAllQuery =  strQuerySelect + " " + strQueryFrom + " " + strQueryWhere + " " + strQueryGroupBy;
203
			strQuery = strQuery + " UNION " + strAllQuery;
204
		}
205

    
206

    
207

    
208
		boolean includeFlatClassifications = state.getConfig().isIncludeFlatClassifications();
209
		//concepts with
210
		if (includeFlatClassifications){
211
			String strFlatQuery =
212
					" SELECT pt.PTRefFk AS secRefFk, r.RefCache AS secRef " +
213
					" FROM PTaxon AS pt LEFT OUTER JOIN " +
214
					          " Reference r ON pt.PTRefFk = r.RefId LEFT OUTER JOIN " +
215
					          " RelPTaxon rel1 ON pt.PTNameFk = rel1.PTNameFk2 AND pt.PTRefFk = rel1.PTRefFk2 LEFT OUTER JOIN " +
216
					          " RelPTaxon AS rel2 ON pt.PTNameFk = rel2.PTNameFk1 AND pt.PTRefFk = rel2.PTRefFk1 " +
217
					" WHERE (rel2.RelQualifierFk IS NULL) AND (rel1.RelQualifierFk IS NULL) " +
218
					" GROUP BY pt.PTRefFk, r.RefCache "
219
					;
220

    
221
			strQuery = strQuery + " UNION " + strFlatQuery;
222
		}
223

    
224

    
225

    
226
		if (state.getConfig().getClassificationQuery() != null){
227
			strQuery = state.getConfig().getClassificationQuery();
228
		}
229
		return strQuery;
230
	}
231

    
232
	@Override
233
	protected String getRecordQuery(BerlinModelImportConfigurator config) {
234
		String strQuery =
235
			" SELECT RelPTaxon.*, fromTaxon.RIdentifier as taxon1Id, toTaxon.RIdentifier as taxon2Id, toTaxon.PTRefFk as treeRefFk, fromTaxon.PTRefFk as fromRefFk, q.is_concept_relation " +
236
			" FROM PTaxon as fromTaxon " +
237
              	" INNER JOIN RelPTaxon ON fromTaxon.PTNameFk = relPTaxon.PTNameFk1 AND fromTaxon.PTRefFk = relPTaxon.PTRefFk1 " +
238
              	" INNER JOIN PTaxon AS toTaxon ON RelPTaxon.PTNameFk2 = ToTaxon.PTNameFk AND RelPTaxon.PTRefFk2 = ToTaxon.PTRefFk " +
239
              	" INNER JOIN RelPTQualifier q ON q.RelPTQualifierId = RelPTaxon.RelQualifierFk " +
240
            " WHERE RelPTaxon.RelPTaxonId IN ("+ID_LIST_TOKEN+") " +
241
            " ORDER BY RelPTaxon.RelPTaxonId ";
242
		return strQuery;
243
	}
244

    
245
	@Override
246
	public boolean doPartition(ResultSetPartitioner partitioner, BerlinModelImportState state) {
247
		boolean success = true ;
248
		Set<TaxonBase> taxaToSave = new HashSet<>();
249
		Map<String, TaxonBase> taxonMap = partitioner.getObjectMap(BerlinModelTaxonImport.NAMESPACE);
250
		Map<Integer, Classification> classificationMap = new HashMap<>();
251
		Map<String, Reference> refMap = partitioner.getObjectMap(BerlinModelReferenceImport.REFERENCE_NAMESPACE);
252

    
253
		ResultSet rs = partitioner.getResultSet();
254

    
255
		try{
256
			int i = 0;
257
			//for each reference
258
			while (rs.next()){
259

    
260
				if ((i++ % modCount) == 0 && i!= 1 ){ logger.info("RelPTaxa handled: " + (i-1));}
261

    
262
				int relPTaxonId = rs.getInt("RelPTaxonId");
263
				Integer taxon1Id = nullSafeInt(rs, "taxon1Id");
264
				Integer taxon2Id = nullSafeInt(rs, "taxon2Id");
265
				Integer ptRefFk1 = nullSafeInt(rs, "PTRefFk1");
266
				Integer ptRefFk2 = nullSafeInt(rs, "PTRefFk2");
267

    
268
				int relQualifierFk = -1;
269
				try {
270
					Integer relRefFk = nullSafeInt(rs,"relRefFk");
271
					int treeRefFk = rs.getInt("treeRefFk");
272
					int fromRefFk = rs.getInt("fromRefFk");
273

    
274
					relQualifierFk = rs.getInt("relQualifierFk");
275
					String notes = rs.getString("notes");
276
					boolean isConceptRelationship = rs.getBoolean("is_concept_relation");
277

    
278
					TaxonBase<?> taxon1 = taxonMap.get(String.valueOf(taxon1Id));
279
					TaxonBase<?> taxon2 = taxonMap.get(String.valueOf(taxon2Id));
280

    
281
					String refFk = String.valueOf(relRefFk);
282
					Reference citation = refMap.get(refFk);
283

    
284
					String microcitation = null; //does not exist in RelPTaxon
285

    
286
					if (taxon2 != null && taxon1 != null){
287
						if (!(taxon2 instanceof Taxon)){
288
							logger.error("ToTaxon (ID = " + taxon2.getId()+ ", RIdentifier = " + taxon2Id + ") can't be casted to Taxon. RelPTaxon: " + relPTaxonId );
289
							success = false;
290
							continue;
291
						}
292
						AnnotatableEntity taxonRelationship = null;
293
						Taxon toTaxon = (Taxon)taxon2;
294
						if (isTaxonRelationship(relQualifierFk)){
295
							if (!(taxon1 instanceof Taxon)){
296
								logger.error("TaxonBase (ID = " + taxon1.getId()+ ", RIdentifier = " + taxon1Id + ") for TaxonRelation ("+relPTaxonId+") can't be casted to Taxon");
297
								success = false;
298
								continue;
299
							}
300
                            if(state.getConfig().isEuroMed() && CdmUtils.nullSafeEqual(relRefFk, ptRefFk2)){
301
                                citation = null;
302
                            }
303
							Taxon fromTaxon = (Taxon)taxon1;
304
							if (relQualifierFk == TAX_REL_IS_INCLUDED_IN){
305
							    taxonRelationship = makeTaxonomicallyIncluded(state, classificationMap, treeRefFk, fromTaxon, toTaxon, citation, microcitation);
306
							}else if (relQualifierFk == TAX_REL_IS_MISAPPLIED_NAME_OF){
307
							    boolean isProParte = "p.p.".equals(notes);
308
							    if (isProParte){
309
								    notes = null;
310
								}
311
							    boolean isDoubtful = "?".equals(notes);
312
                                if (isDoubtful){
313
                                    notes = null;
314
                                }
315

    
316
                                if (notes!= null && notes.startsWith("{non ") && notes.endsWith("}")){
317
                                    notes = notes.substring(1, notes.length() - 1);
318
                                }
319
							    //handle auct. author
320
						        if (fromTaxon.getSec() == null || fromTaxon.getSec().getTitleCache().startsWith("auct.")){
321
							        String existingSecTitle = fromTaxon.getSec() == null ? null : fromTaxon.getSec().getTitleCache().trim();
322
						            String existingAppendedPhrase = fromTaxon.getAppendedPhrase();
323
							        if (fromTaxon.getSec() == null && isBlank(existingAppendedPhrase)){
324
							            existingAppendedPhrase = "auct.";
325
							        }
326
						            fromTaxon.setSec(null);
327
							        if (isNotBlank(existingAppendedPhrase) && isNotBlank(notes)){
328
							            logger.warn("Misapplied name has >1 MA relation with a note, RelId: " + relPTaxonId);
329
							        }
330

    
331
							        String newAppendedPhrase = CdmUtils.concat(", ", existingSecTitle, notes);
332
							        fromTaxon.setAppendedPhrase(CdmUtils.concat("; ", existingAppendedPhrase, newAppendedPhrase));
333
							        if (isBlank(fromTaxon.getAppendedPhrase())){
334
							            logger.warn("Appended phrase is empty. This is probably not correct. RelID: " + relPTaxonId);
335
							        }else if ("auct.".equals(fromTaxon.getAppendedPhrase())){
336
							            fromTaxon.setAppendedPhrase(null);
337
							        }
338
							        notes = null;
339
							    }else if (notes != null && notes.startsWith("non ")){
340
							        fromTaxon.setAppendedPhrase(CdmUtils.concat(", ", fromTaxon.getAppendedPhrase(), notes));
341
							        notes = null;
342
							    }
343

    
344
							    if (isProParte){
345
							        taxonRelationship = toTaxon.addProParteMisappliedName(fromTaxon, citation, microcitation);
346
							    }else{
347
							        taxonRelationship = toTaxon.addMisappliedName(fromTaxon, citation, microcitation);
348
	                            }
349
							    if (isDoubtful){
350
							        ((TaxonRelationship)taxonRelationship).setDoubtful(true);
351
							    }
352
                            }else if (relQualifierFk == TAX_REL_IS_PROPARTE_SYN_OF ||
353
                                    relQualifierFk == TAX_REL_IS_PROPARTE_HOMOTYPIC_SYNONYM_OF ||
354
                                    relQualifierFk == TAX_REL_IS_PROPARTE_HETEROTYPIC_SYNONYM_OF ){
355
                                if(relQualifierFk == TAX_REL_IS_PROPARTE_HOMOTYPIC_SYNONYM_OF){
356
                                    logger.warn("Homotypic pro parte synonyms not handled in CDM. Please add homotypie manually. RelPTID: " +  relPTaxonId);
357
                                }
358
                                //heterotypic we expect as the normal relationship, don't store explicitly
359
                                toTaxon.addProparteSynonym(fromTaxon, citation, microcitation);
360
                            }else if(relQualifierFk == TAX_REL_IS_PARTIAL_SYN_OF ||
361
                                    relQualifierFk == TAX_REL_IS_PARTIAL_HOMOTYPIC_SYNONYM_OF ||
362
                                    relQualifierFk == TAX_REL_IS_PARTIAL_HETEROTYPIC_SYNONYM_OF ){
363
                                if (relQualifierFk == TAX_REL_IS_PARTIAL_HOMOTYPIC_SYNONYM_OF ||
364
                                        relQualifierFk == TAX_REL_IS_PARTIAL_HETEROTYPIC_SYNONYM_OF){
365
                                    logger.warn("Homotypie and heterotypie not yet handled for partial synonyms");
366
                                }
367
                                toTaxon.addPartialSynonym(fromTaxon, citation, microcitation);
368
                            }else{
369
								handleAllRelatedTaxa(state, fromTaxon, classificationMap, fromRefFk);
370
								handleAllRelatedTaxa(state, toTaxon, classificationMap, treeRefFk);
371
								logger.warn("Unhandled taxon relationship: RelId:" + relPTaxonId + "; QualifierId: " + relQualifierFk);
372
							}
373
						}else if (isSynonymRelationship(relQualifierFk)){
374
							if (!(taxon1 instanceof Synonym)){
375
								logger.warn("Validated: Taxon (ID = " + taxon1.getId()+ ", RIdentifier = " + taxon1Id + ") can't be casted to Synonym");
376
								success = false;
377
								continue;
378
							}
379
							handleAllRelatedTaxa(state, toTaxon, classificationMap, treeRefFk);
380
							Synonym synonym = (Synonym)taxon1;
381
							if (synonym.getAcceptedTaxon()!= null){
382
							    logger.warn("RelID: " + relPTaxonId + ". Synonym ("+taxon1Id +") already has an accepted taxon. Create clone.");
383
							    synonym = (Synonym)synonym.clone();
384
							}
385
							makeSynRel(state, relQualifierFk, toTaxon, synonym, citation, microcitation);
386

    
387
							if (relQualifierFk == TAX_REL_IS_SYNONYM_OF ||
388
									relQualifierFk == TAX_REL_IS_HOMOTYPIC_SYNONYM_OF ||
389
									relQualifierFk == TAX_REL_IS_HETEROTYPIC_SYNONYM_OF){
390
							}else{
391
								success = false;
392
								logger.warn("Synonym relationship type not yet implemented: " + relQualifierFk);
393
							}
394
							//
395
							notes = handleSynonymNotes(state, toTaxon, synonym, notes, relPTaxonId);
396
						}else if (isConceptRelationship){
397
							ResultWrapper<Boolean> isInverse = ResultWrapper.NewInstance(false);
398
							ResultWrapper<Boolean> isDoubtful = ResultWrapper.NewInstance(false);
399
							try {
400
								TaxonRelationshipType relType = BerlinModelTransformer.taxonRelId2TaxonRelType(relQualifierFk, isInverse, isDoubtful);
401

    
402
								if (! (taxon1 instanceof Taxon)){
403
									success = false;
404
									logger.error("TaxonBase (ID = " + taxon1.getId()+ ", RIdentifier = " + taxon1Id + ") can't be casted to Taxon");
405
								}else{
406
									Taxon fromTaxon = (Taxon)taxon1;
407
									if (isInverse.getValue() == true){
408
										Taxon tmp = fromTaxon;
409
										fromTaxon = toTaxon;
410
										toTaxon = tmp;
411
									}
412
									taxonRelationship = fromTaxon.addTaxonRelation(toTaxon, relType, citation, microcitation);
413
									handleAllRelatedTaxa(state, toTaxon, classificationMap, treeRefFk);
414
									handleAllRelatedTaxa(state, fromTaxon, classificationMap, fromRefFk);
415
									if (isDoubtful.getValue() == true){
416
										((TaxonRelationship)taxonRelationship).setDoubtful(true);
417
									}
418
								}
419
							} catch (UnknownCdmTypeException e) {
420
								logger.warn("TaxonRelationShipType " + relQualifierFk + " (conceptRelationship) not yet implemented");
421
								 success = false;
422
							}
423
						}else {
424
							logger.warn("TaxonRelationShipType " + relQualifierFk + " not yet implemented: RelPTaxonId = " + relPTaxonId );
425
							success = false;
426
						}
427

    
428
						if (taxonRelationship != null && isNotBlank(notes)){
429
						    doNotes(taxonRelationship, notes);
430
						}
431
						if (isNotBlank(notes)){
432
						    logger.warn("Notes in RelPTaxon should all be handled explicitly and should not exist as notes anymore. RelID: " + relPTaxonId + ". Note: " + notes);
433
						}
434
						taxaToSave.add(taxon2);
435

    
436
						//TODO
437
						//etc.
438
					}else{
439
						if (taxon2 != null && taxon1 == null){
440
							logger.warn("First taxon ("+taxon1Id+") for RelPTaxon " + relPTaxonId + " does not exist in store. RelType: " + relQualifierFk);
441
						}else if (taxon2 == null && taxon1 != null){
442
							logger.warn("Second taxon ("+taxon2Id +") for RelPTaxon " + relPTaxonId + " does not exist in store. RelType: " + relQualifierFk);
443
						}else{
444
							logger.warn("Both taxa ("+taxon1Id+","+taxon2Id +") for RelPTaxon " + relPTaxonId + " do not exist in store. RelType: " + relQualifierFk);
445
						}
446

    
447
						success = false;
448
					}
449
				} catch (Exception e) {
450
					logger.error("Exception occurred when trying to handle taxon relationship " + relPTaxonId + " relQualifierFK " + relQualifierFk + " (" + taxon1Id + ","+ taxon2Id + "): " + e.getMessage());
451
					e.printStackTrace();
452
				}
453
			}
454
		}catch(SQLException e){
455
			throw new RuntimeException(e);
456
		}
457
		logger.info("Taxa to save: " + taxaToSave.size());
458
		partitioner.startDoSave();
459
		getTaxonService().saveOrUpdate(taxaToSave);
460
		classificationMap = null;
461
		taxaToSave = null;
462

    
463
		return success;
464
	}
465

    
466

    
467
	/**
468
     * @param toTaxon
469
     * @param synonym
470
     * @param notes
471
     * @return
472
     */
473
    private String handleSynonymNotes(BerlinModelImportState state, Taxon toTaxon, Synonym synonym, String notes, int relId) {
474
        if (state.getConfig().isEuroMed() && isNotBlank(notes)){
475
            notes = notes.trim();
476
            if (notes.startsWith("[non ") && notes.endsWith("]")){
477
                notes = notes.substring(5, notes.length()-1).trim();
478
                String[] splits = notes.split(", nec ");
479
                for (String split : splits){
480
                    String nameStr = split.replace("<i>", "").replace("</i>", "");
481
                    NonViralNameParserImpl parser = NonViralNameParserImpl.NewInstance();
482
                    TaxonName name;
483
                    NomenclaturalStatusType status = null;
484
                    if (nameStr.endsWith(", nom. rej.") || nameStr.endsWith(", nom. cons.")||nameStr.endsWith(", nom. illeg.")){
485
                        String statusStr = nameStr.endsWith(", nom. rej.")? ", nom. rej.":
486
                            nameStr.endsWith(", nom. cons.")? ", nom. cons.":
487
                            ", nom. illeg.";
488
                        nameStr = nameStr.replace(statusStr, "");
489
                        statusStr = statusStr.replace(", ", "");
490
                        try {
491
                            status = NomenclaturalStatusType.getNomenclaturalStatusTypeByAbbreviation(statusStr, null);
492
                        } catch (UnknownCdmTypeException e) {
493
                            logger.warn("NomStatusType not recognized: "+  statusStr + ", RelId: " +  relId);
494
                        }
495
                    }
496

    
497
                    if (nameStr.contains(",") || nameStr.contains(" in ") ){
498
                        name = parser.parseReferencedName(nameStr, state.getConfig().getNomenclaturalCode(), null);
499
                    }else if (nameStr.matches(".*\\s\\d{4}")){
500
                        String nameStr2 = nameStr.substring(0, nameStr.length() - 5).trim();
501
                        String yearStr = nameStr.substring(nameStr.length()-4);
502
                        name = (TaxonName)parser.parseFullName(nameStr2, state.getConfig().getNomenclaturalCode(), null);
503
                        Reference nomRef = name.getNomenclaturalReference();
504
                        if (nomRef == null){
505
                            nomRef = ReferenceFactory.newGeneric();
506
                            name.setNomenclaturalReference(nomRef);
507
                        }
508
                        nomRef.setDatePublished(TimePeriodParser.parseStringVerbatim(yearStr));
509
                    }else if (nameStr.endsWith(" 1831-1832")){
510
                        String nameStr2 = nameStr.substring(0, nameStr.length() - 10).trim();
511
                        name = (TaxonName)parser.parseFullName(nameStr2, state.getConfig().getNomenclaturalCode(), null);
512
                        Reference nomRef = name.getNomenclaturalReference();
513
                        if (nomRef == null){
514
                            nomRef = ReferenceFactory.newGeneric();
515
                            name.setNomenclaturalReference(nomRef);
516
                        }
517
                        nomRef.setDatePublished(TimePeriodParser.parseStringVerbatim("1831-1832"));
518
                    }else{
519
                        name = parser.parseReferencedName(nameStr, state.getConfig().getNomenclaturalCode(), null);
520
                    }
521
                    if (name.isProtectedTitleCache() || name.isProtectedNameCache()
522
                            || name.getNomenclaturalReference() != null && (name.getNomenclaturalReference().isProtectedAbbrevTitleCache()|| name.getNomenclaturalReference().isProtectedTitleCache() )){
523
                        logger.warn("Blocking name for synonym relation could not be parsed: " + nameStr + ", RelId: "+ relId);
524
                    }
525
                    if (status != null){
526
                        name.addStatus(NomenclaturalStatus.NewInstance(status));
527
                    }
528
                    synonym.getName().addRelationshipFromName(name, NameRelationshipType.BLOCKING_NAME_FOR(), null, null, null);
529

    
530
                    getNameService().saveOrUpdate(name);
531
                }
532
                return null;
533
            }else{
534
                return notes;
535
            }
536
        }else{
537
            return notes;
538
        }
539
    }
540

    
541
    private void handleAllRelatedTaxa(BerlinModelImportState state, Taxon taxon,
542
            Map<Integer, Classification> classificationMap, Integer secRefFk) {
543
		if (taxon.getTaxonNodes().size() > 0){
544
			return;
545
		}else{
546
			Classification classification = getClassificationTree(state, classificationMap, secRefFk);
547
			classification.addChildTaxon(taxon, null, null);
548
		}
549
	}
550

    
551
	@Override
552
	protected void doInvoke(BerlinModelImportState state){
553
		try {
554
			makeClassifications(state);
555
			super.doInvoke(state);
556
			makeFlatClassificationTaxa(state);
557
			return;
558
		} catch (SQLException e) {
559
			throw new RuntimeException(e);
560
		}
561

    
562
	}
563

    
564

    
565
	private void makeFlatClassificationTaxa(BerlinModelImportState state) {
566
		//Note: this part still does not use partitions
567
		logger.info("Flat classifications start");
568
		TransactionStatus txStatus = startTransaction();
569
		if (! state.getConfig().isIncludeFlatClassifications()){
570
			return;
571
		}
572
		String sql = " SELECT pt.PTRefFk AS secRefFk, pt.RIdentifier " +
573
						" FROM PTaxon AS pt " +
574
							" LEFT OUTER JOIN RelPTaxon ON pt.PTNameFk = RelPTaxon.PTNameFk2 AND pt.PTRefFk = RelPTaxon.PTRefFk2 " +
575
							"  LEFT OUTER JOIN RelPTaxon AS RelPTaxon_1 ON pt.PTNameFk = RelPTaxon_1.PTNameFk1 AND pt.PTRefFk = RelPTaxon_1.PTRefFk1 " +
576
						" WHERE (RelPTaxon_1.RelQualifierFk IS NULL) AND (dbo.RelPTaxon.RelQualifierFk IS NULL) " +
577
						" ORDER BY pt.PTRefFk "	;
578
		ResultSet rs = state.getConfig().getSource().getResultSet(sql);
579
		Map<Object, Map<String, ? extends CdmBase>> maps = getRelatedObjectsForFlatPartition(rs);
580

    
581
		Map<String, TaxonBase> taxonMap = (Map<String, TaxonBase>) maps.get(BerlinModelTaxonImport.NAMESPACE);
582
		Map<Integer, Classification> classificationMap = new HashMap<>();
583

    
584
		rs = state.getConfig().getSource().getResultSet(sql);
585
		try {
586
			while (rs.next()){
587
				Integer treeRefFk = rs.getInt("secRefFk");
588
				String taxonId = rs.getString("RIdentifier");
589
				Classification classification = getClassificationTree(state, classificationMap, treeRefFk);
590
				TaxonBase<?> taxon = taxonMap.get(taxonId);
591
				if (taxon == null){
592
					String message = "TaxonBase for taxon id (%s) not found in taxonMap";
593
					logger.warn(String.format(message, taxonId, taxonId));
594
				}else if (taxon.isInstanceOf(Taxon.class)){
595
					classification.addChildTaxon(CdmBase.deproxy(taxon, Taxon.class), null, null);
596
				}else{
597
					String message = "TaxonBase for taxon is not of class Taxon but %s (RIdentifier %s)";
598
					logger.warn(String.format(message, taxon.getClass(), taxonId));
599
				}
600
			}
601
		} catch (SQLException e) {
602
			e.printStackTrace();
603
		}
604
		commitTransaction(txStatus);
605
		logger.info("Flat classifications end");
606

    
607
	}
608

    
609
	@Override
610
	protected String getIdQuery(BerlinModelImportState state) {
611
		if (state.getConfig().getRelTaxaIdQuery() != null){
612
			return state.getConfig().getRelTaxaIdQuery();
613
		}else{
614
			return super.getIdQuery(state);
615
		}
616
	}
617

    
618
	@Override
619
	public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs, BerlinModelImportState state) {
620
		String nameSpace;
621
		Class<?> cdmClass;
622
		Set<String> idSet;
623
		Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<>();
624

    
625
		try{
626
			Set<String> taxonIdSet = new HashSet<>();
627
			Set<String> referenceIdSet = new HashSet<>();
628
//			Set<String> classificationIdSet = new HashSet<>();
629
			while (rs.next()){
630
				handleForeignKey(rs, taxonIdSet, "taxon1Id");
631
				handleForeignKey(rs, taxonIdSet, "taxon2Id");
632
//				handleForeignKey(rs, classificationIdSet, "treeRefFk");
633
				handleForeignKey(rs, referenceIdSet, "RelRefFk");
634
	}
635

    
636
			//taxon map
637
			nameSpace = BerlinModelTaxonImport.NAMESPACE;
638
			cdmClass = TaxonBase.class;
639
			idSet = taxonIdSet;
640
			Map<String, TaxonBase> taxonMap = (Map<String, TaxonBase>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
641
			result.put(nameSpace, taxonMap);
642

    
643
//			//tree map
644
//			nameSpace = "Classification";
645
//			cdmClass = Classification.class;
646
//			idSet = classificationIdSet;
647
//			Map<String, Classification> treeMap = (Map<String, Classification>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
648
//			result.put(cdmClass, treeMap);
649
//			Set<UUID> treeUuidSet = state
650
//			getClassificationService().find(uuidSet);
651
//
652
			//reference map
653
			nameSpace = BerlinModelReferenceImport.REFERENCE_NAMESPACE;
654
			cdmClass = Reference.class;
655
			idSet = referenceIdSet;
656
			Map<String, Reference> referenceMap = (Map<String, Reference>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
657
			result.put(nameSpace, referenceMap);
658

    
659
		} catch (SQLException e) {
660
			throw new RuntimeException(e);
661
		}
662
		return result;
663
	}
664

    
665

    
666
	private Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForFlatPartition( ResultSet rs) {
667
		String nameSpace;
668
		Class cdmClass;
669
		Set<String> idSet;
670
		Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<>();
671

    
672
		try{
673
			Set<String> taxonIdSet = new HashSet<>();
674
			Set<String> referenceIdSet = new HashSet<>();
675
//			Set<String> classificationIdSet = new HashSet<>();
676
			while (rs.next()){
677
				handleForeignKey(rs, taxonIdSet, "RIdentifier");
678
//				handleForeignKey(rs, classificationIdSet, "treeRefFk");
679
			}
680

    
681
			//taxon map
682
			nameSpace = BerlinModelTaxonImport.NAMESPACE;
683
			cdmClass = TaxonBase.class;
684
			idSet = taxonIdSet;
685
			Map<String, TaxonBase> taxonMap = (Map<String, TaxonBase>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
686
			result.put(nameSpace, taxonMap);
687

    
688
//			//tree map
689
//			nameSpace = "Classification";
690
//			cdmClass = Classification.class;
691
//			idSet = classificationIdSet;
692
//			Map<String, Classification> treeMap = (Map<String, Classification>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
693
//			result.put(cdmClass, treeMap);
694
//			Set<UUID> treeUuidSet = state
695
//			getClassificationService().find(uuidSet);
696
//
697

    
698
		} catch (SQLException e) {
699
			throw new RuntimeException(e);
700
		}
701
		return result;
702
	}
703

    
704

    
705
	private void makeSynRel (BerlinModelImportState state, int relQualifierFk, Taxon toTaxon, Synonym synonym, Reference citation, String microcitation){
706
		if (state.getConfig().isWarnForDifferingSynonymReference() && citation != null && !citation.equals(synonym.getSec())){
707
		    logger.warn("A synonym relationship citation is given and differs from synonym secundum. This can not be handled in CDM");
708
		}
709
		if (isNotBlank(microcitation)  && !microcitation.equals(synonym.getSecMicroReference())){
710
            logger.warn("A synonym relationship microcitation is given and differs from synonym secundum micro reference. This can not be handled in CDM");
711
        }
712
	    if (relQualifierFk == TAX_REL_IS_HOMOTYPIC_SYNONYM_OF ||
713
				relQualifierFk == TAX_REL_IS_PROPARTE_HOMOTYPIC_SYNONYM_OF ||
714
				relQualifierFk == TAX_REL_IS_PARTIAL_HOMOTYPIC_SYNONYM_OF){
715
			toTaxon.addHomotypicSynonym(synonym);
716
		}else if (relQualifierFk == TAX_REL_IS_HETEROTYPIC_SYNONYM_OF ||
717
				relQualifierFk == TAX_REL_IS_PROPARTE_HETEROTYPIC_SYNONYM_OF ||
718
				relQualifierFk == TAX_REL_IS_PARTIAL_HETEROTYPIC_SYNONYM_OF){
719
			toTaxon.addSynonym(synonym, SynonymType.HETEROTYPIC_SYNONYM_OF());
720
		}else if (relQualifierFk == TAX_REL_IS_SYNONYM_OF ||
721
				relQualifierFk == TAX_REL_IS_PROPARTE_SYN_OF ||
722
				relQualifierFk == TAX_REL_IS_PARTIAL_SYN_OF){
723
			toTaxon.addSynonym(synonym, SynonymType.SYNONYM_OF());
724
		}else{
725
			logger.warn("SynonymyRelationShipType could not be defined for relQualifierFk " + relQualifierFk + ". 'Unknown'-Type taken instead.");
726
			toTaxon.addSynonym(synonym, SynonymType.SYNONYM_OF());
727
		}
728
		return;
729

    
730
	}
731

    
732
	private  boolean isSynonymRelationship(int relQualifierFk){
733
		if (relQualifierFk == TAX_REL_IS_SYNONYM_OF ||
734
			relQualifierFk == TAX_REL_IS_HOMOTYPIC_SYNONYM_OF ||
735
			relQualifierFk == TAX_REL_IS_HETEROTYPIC_SYNONYM_OF ||
736
			relQualifierFk == TAX_REL_IS_PROPARTE_SYN_OF ||
737
			relQualifierFk == TAX_REL_IS_PARTIAL_SYN_OF ||
738
			relQualifierFk == TAX_REL_IS_PROPARTE_HOMOTYPIC_SYNONYM_OF ||
739
			relQualifierFk == TAX_REL_IS_PROPARTE_HETEROTYPIC_SYNONYM_OF ||
740
			relQualifierFk == TAX_REL_IS_PARTIAL_HOMOTYPIC_SYNONYM_OF ||
741
			relQualifierFk == TAX_REL_IS_PARTIAL_HETEROTYPIC_SYNONYM_OF
742
		){
743
			return true;
744
		}else{
745
			return false;
746
		}
747
	}
748

    
749
	private  boolean isTaxonRelationship(int relQualifierFk){
750
		if (relQualifierFk == TAX_REL_IS_INCLUDED_IN ||
751
				relQualifierFk == TAX_REL_IS_MISAPPLIED_NAME_OF){
752
			return true;
753
		}else if (relQualifierFk == TAX_REL_IS_PROPARTE_SYN_OF ||
754
	                relQualifierFk == TAX_REL_IS_PARTIAL_SYN_OF ||
755
	                relQualifierFk == TAX_REL_IS_PROPARTE_HOMOTYPIC_SYNONYM_OF ||
756
	                relQualifierFk == TAX_REL_IS_PROPARTE_HETEROTYPIC_SYNONYM_OF ||
757
	                relQualifierFk == TAX_REL_IS_PARTIAL_HOMOTYPIC_SYNONYM_OF ||
758
	                relQualifierFk == TAX_REL_IS_PARTIAL_HETEROTYPIC_SYNONYM_OF
759
                    ){
760
	            return true;
761

    
762
		}else{
763
			return false;
764
		}
765
	}
766

    
767
	private TaxonNode makeTaxonomicallyIncluded(BerlinModelImportState state, Map<Integer, Classification> classificationMap,
768
	        int treeRefFk, Taxon child, Taxon parent, Reference citation, String microCitation){
769
		Classification tree = getClassificationTree(state, classificationMap, treeRefFk);
770
		return tree.addParentChild(parent, child, citation, microCitation);
771
	}
772

    
773
	private Classification getClassificationTree(BerlinModelImportState state, Map<Integer, Classification> classificationMap, int treeRefFk) {
774
		if (state.getConfig().isUseSingleClassification()){
775
			if (state.getConfig().getSourceSecId() != null){
776
				treeRefFk = (Integer)state.getConfig().getSourceSecId();
777
			}else{
778
				treeRefFk = 1;
779
			}
780

    
781
		}
782
		Classification tree = classificationMap.get(treeRefFk);
783
		if (tree == null){
784
			UUID treeUuid = state.getTreeUuidByIntTreeKey(treeRefFk);
785
			if (treeUuid == null){
786
				throw new IllegalStateException("treeUUID does not exist in state for " + treeRefFk );
787
			}
788
			tree = getClassificationService().find(treeUuid);
789
			classificationMap.put(treeRefFk, tree);
790
		}
791
		if (tree == null){
792
			throw new IllegalStateException("Tree for ToTaxon reference " + treeRefFk + " does not exist.");
793
		}
794
		return tree;
795
	}
796

    
797
	@Override
798
	protected boolean doCheck(BerlinModelImportState state){
799
		IOValidator<BerlinModelImportState> validator = new BerlinModelTaxonRelationImportValidator();
800
		return validator.validate(state);
801
	}
802

    
803
	@Override
804
	protected boolean isIgnore(BerlinModelImportState state){
805
		return ! state.getConfig().isDoRelTaxa();
806
	}
807

    
808

    
809
}
(18-18/22)