Project

General

Profile

Download (35.2 KB) Statistics
| Branch: | Revision:
1
/**
2
* Copyright (C) 2007 EDIT
3
* European Distributed Institute of Taxonomy
4
* http://www.e-taxonomy.eu
5
*
6
* The contents of this file are subject to the Mozilla Public License Version 1.1
7
* See LICENSE.TXT at the top of this package for the full license terms.
8
*/
9

    
10
package eu.etaxonomy.cdm.io.berlinModel.in;
11

    
12
import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.TAX_REL_IS_HETEROTYPIC_SYNONYM_OF;
13
import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.TAX_REL_IS_HOMOTYPIC_SYNONYM_OF;
14
import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.TAX_REL_IS_INCLUDED_IN;
15
import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.TAX_REL_IS_MISAPPLIED_NAME_OF;
16
import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.TAX_REL_IS_PARTIAL_HETEROTYPIC_SYNONYM_OF;
17
import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.TAX_REL_IS_PARTIAL_HOMOTYPIC_SYNONYM_OF;
18
import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.TAX_REL_IS_PARTIAL_SYN_OF;
19
import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.TAX_REL_IS_PROPARTE_HETEROTYPIC_SYNONYM_OF;
20
import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.TAX_REL_IS_PROPARTE_HOMOTYPIC_SYNONYM_OF;
21
import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.TAX_REL_IS_PROPARTE_SYN_OF;
22
import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.TAX_REL_IS_SYNONYM_OF;
23

    
24
import java.sql.ResultSet;
25
import java.sql.SQLException;
26
import java.util.HashMap;
27
import java.util.HashSet;
28
import java.util.Map;
29
import java.util.Set;
30
import java.util.UUID;
31

    
32
import org.apache.commons.lang.StringUtils;
33
import org.apache.log4j.Logger;
34
import org.springframework.stereotype.Component;
35
import org.springframework.transaction.TransactionStatus;
36

    
37
import eu.etaxonomy.cdm.common.CdmUtils;
38
import eu.etaxonomy.cdm.common.ResultWrapper;
39
import eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer;
40
import eu.etaxonomy.cdm.io.berlinModel.in.validation.BerlinModelTaxonRelationImportValidator;
41
import eu.etaxonomy.cdm.io.common.IOValidator;
42
import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
43
import eu.etaxonomy.cdm.io.common.Source;
44
import eu.etaxonomy.cdm.model.common.AnnotatableEntity;
45
import eu.etaxonomy.cdm.model.common.CdmBase;
46
import eu.etaxonomy.cdm.model.common.IdentifiableSource;
47
import eu.etaxonomy.cdm.model.name.NameRelationshipType;
48
import eu.etaxonomy.cdm.model.name.NomenclaturalStatus;
49
import eu.etaxonomy.cdm.model.name.NomenclaturalStatusType;
50
import eu.etaxonomy.cdm.model.name.TaxonName;
51
import eu.etaxonomy.cdm.model.reference.Reference;
52
import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
53
import eu.etaxonomy.cdm.model.taxon.Classification;
54
import eu.etaxonomy.cdm.model.taxon.Synonym;
55
import eu.etaxonomy.cdm.model.taxon.SynonymType;
56
import eu.etaxonomy.cdm.model.taxon.Taxon;
57
import eu.etaxonomy.cdm.model.taxon.TaxonBase;
58
import eu.etaxonomy.cdm.model.taxon.TaxonNode;
59
import eu.etaxonomy.cdm.model.taxon.TaxonRelationship;
60
import eu.etaxonomy.cdm.model.taxon.TaxonRelationshipType;
61
import eu.etaxonomy.cdm.strategy.exceptions.UnknownCdmTypeException;
62
import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
63
import eu.etaxonomy.cdm.strategy.parser.TimePeriodParser;
64

    
65
/**
66
 * @author a.mueller
67
 * @since 20.03.2008
68
 */
69
@Component
70
public class BerlinModelTaxonRelationImport  extends BerlinModelImportBase  {
71
    private static final long serialVersionUID = -7234926279240842557L;
72

    
73
    private static final Logger logger = Logger.getLogger(BerlinModelTaxonRelationImport.class);
74

    
75
	public static final String TREE_NAMESPACE = "PTRefFk";
76
	private static final Integer AUCT_REF_ID = 5959;
77

    
78
	private static int modCount = 30000;
79
	private static final String pluralString = "taxon relations";
80
	private static final String dbTableName = "RelPTaxon";
81

    
82

    
83
	public BerlinModelTaxonRelationImport(){
84
		super(dbTableName, pluralString);
85
	}
86

    
87
	/**
88
	 * Creates a classification for each PTaxon reference which belongs to a taxon that is
89
	 * included at least in one <i>taxonomically included</i> relationship.
90
	 *
91
	 * @param state
92
	 * @throws SQLException
93
	 */
94
	private void makeClassifications(BerlinModelImportState state) throws SQLException{
95
		logger.info("start make classification ...");
96

    
97
		Set<String> idSet = getTreeReferenceIdSet(state);
98

    
99
		//reference map
100
		String nameSpace = BerlinModelReferenceImport.REFERENCE_NAMESPACE;
101
		Class<?> cdmClass = Reference.class;
102
		@SuppressWarnings("unchecked")
103
        Map<String, Reference> refMap = (Map<String, Reference>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
104

    
105
		String classificationName = "Classification - No Name";
106

    
107
		ResultSet rs = state.getConfig().getSource().getResultSet(getClassificationQuery(state)) ;
108
		int i = 0;
109
		//for each reference
110
		try {
111
			//TODO handle case useSingleClassification = true && sourceSecId = null, which returns no record
112
			boolean isFirst = true;
113
		    while (rs.next()){
114

    
115
				try {
116
					if ((i++ % modCount) == 0 && i!= 1 ){ logger.info("RelPTaxa handled: " + (i-1));}
117

    
118
					Integer ptRefFkInt = nullSafeInt(rs,"PTRefFk");
119
					String ptRefFk= String.valueOf(ptRefFkInt);
120
					Reference ref = refMap.get(ptRefFk);
121

    
122
					String refCache = rs.getString("RefCache");
123
					if (isNotBlank(refCache)){
124
						classificationName = refCache;
125
					}
126
					if (ref != null && StringUtils.isNotBlank(ref.getTitleCache())){
127
						classificationName = ref.getTitleCache();
128
					}
129
					if (isFirst && isNotBlank(state.getConfig().getClassificationName())){
130
					    classificationName = state.getConfig().getClassificationName();
131
					}
132
					Classification tree = Classification.NewInstance(classificationName);
133
					tree.setReference(ref);
134
					if (i == 1 && state.getConfig().getClassificationUuid() != null){
135
						tree.setUuid(state.getConfig().getClassificationUuid());
136
					}
137
					IdentifiableSource identifiableSource = IdentifiableSource.NewDataImportInstance(ptRefFk, TREE_NAMESPACE);
138
					tree.addSource(identifiableSource);
139

    
140
					getClassificationService().save(tree);
141
					state.putClassificationUuidInt(ptRefFkInt, tree);
142
					isFirst = false;
143
				} catch (Exception e) {
144
					logger.error("Error in BerlinModleTaxonRelationImport.makeClassifications: " + e.getMessage());
145
					e.printStackTrace();
146
				}
147
			}
148
		} catch (SQLException e) {
149
			logger.error("Error in BerlinModleTaxonRelationImport.makeClassifications: " + e.getMessage());
150
			throw e;
151
		}
152
		logger.info("end make classification ...");
153

    
154
		return;
155
	}
156

    
157
	/**
158
	 * @return
159
	 * @throws SQLException
160
	 */
161
	private Set<String> getTreeReferenceIdSet(BerlinModelImportState state) throws SQLException {
162
		Source source = state.getConfig().getSource();
163
		Set<String> result = new HashSet<String>();
164
		ResultSet rs = source.getResultSet(getClassificationQuery(state)) ;
165
		while (rs.next()){
166
			Object id = rs.getObject("PTRefFk");
167
			result.add(String.valueOf(id));
168
		}
169
		return result;
170
	}
171

    
172
	private String getClassificationQuery(BerlinModelImportState state) {
173
		boolean includeAllClassifications = state.getConfig().isIncludeAllNonMisappliedRelatedClassifications();
174
		String strQuerySelect = "SELECT PTaxon.PTRefFk, r.RefCache ";
175
		String strQueryFrom = " FROM RelPTaxon " +
176
							" INNER JOIN PTaxon AS PTaxon ON RelPTaxon.PTNameFk2 = PTaxon.PTNameFk AND RelPTaxon.PTRefFk2 = PTaxon.PTRefFk " +
177
							" INNER JOIN Reference r ON PTaxon.PTRefFk = r.RefId ";
178
		String strQueryWhere = " WHERE (RelPTaxon.RelQualifierFk = 1) ";
179
		if (includeAllClassifications){
180
			strQueryWhere = " WHERE (RelPTaxon.RelQualifierFk <> 3) ";
181
		}else{
182
			if (state.getConfig().isUseSingleClassification()){
183
				if (state.getConfig().getSourceSecId()!= null){
184
					strQueryWhere += " AND PTaxon.PTRefFk = " + state.getConfig().getSourceSecId() +  " ";
185
				}else{
186
					strQueryWhere += " AND (1=0) ";
187
				}
188
			}
189
		}
190

    
191
		String strQueryGroupBy = " GROUP BY PTaxon.PTRefFk, r.RefCache ";
192
		String strQuery = strQuerySelect + " " + strQueryFrom + " " + strQueryWhere + " " + strQueryGroupBy;
193

    
194

    
195
		if (includeAllClassifications){
196
			//add otherdirection
197
			strQuerySelect = "SELECT PTaxon.PTRefFk, r.RefCache ";
198
			strQueryFrom = " FROM RelPTaxon rel " +
199
								" INNER JOIN PTaxon AS PTaxon ON rel.PTNameFk1 = PTaxon.PTNameFk AND rel.PTRefFk1 = PTaxon.PTRefFk " +
200
								" INNER JOIN Reference r ON PTaxon.PTRefFk = r.RefId ";
201
			strQueryWhere =" WHERE (rel.RelQualifierFk <> 3) ";
202
			String strAllQuery =  strQuerySelect + " " + strQueryFrom + " " + strQueryWhere + " " + strQueryGroupBy;
203
			strQuery = strQuery + " UNION " + strAllQuery;
204
		}
205

    
206

    
207

    
208
		boolean includeFlatClassifications = state.getConfig().isIncludeFlatClassifications();
209
		//concepts with
210
		if (includeFlatClassifications){
211
			String strFlatQuery =
212
					" SELECT pt.PTRefFk AS secRefFk, r.RefCache AS secRef " +
213
					" FROM PTaxon AS pt LEFT OUTER JOIN " +
214
					          " Reference r ON pt.PTRefFk = r.RefId LEFT OUTER JOIN " +
215
					          " RelPTaxon rel1 ON pt.PTNameFk = rel1.PTNameFk2 AND pt.PTRefFk = rel1.PTRefFk2 LEFT OUTER JOIN " +
216
					          " RelPTaxon AS rel2 ON pt.PTNameFk = rel2.PTNameFk1 AND pt.PTRefFk = rel2.PTRefFk1 " +
217
					" WHERE (rel2.RelQualifierFk IS NULL) AND (rel1.RelQualifierFk IS NULL) " +
218
					" GROUP BY pt.PTRefFk, r.RefCache "
219
					;
220

    
221
			strQuery = strQuery + " UNION " + strFlatQuery;
222
		}
223

    
224

    
225

    
226
		if (state.getConfig().getClassificationQuery() != null){
227
			strQuery = state.getConfig().getClassificationQuery();
228
		}
229
		return strQuery;
230
	}
231

    
232
	@Override
233
	protected String getRecordQuery(BerlinModelImportConfigurator config) {
234
		String strQuery =
235
			" SELECT RelPTaxon.*, fromTaxon.RIdentifier as taxon1Id, toTaxon.RIdentifier as taxon2Id, toTaxon.PTRefFk as treeRefFk, fromTaxon.PTRefFk as fromRefFk, q.is_concept_relation " +
236
			" FROM PTaxon as fromTaxon " +
237
              	" INNER JOIN RelPTaxon ON fromTaxon.PTNameFk = relPTaxon.PTNameFk1 AND fromTaxon.PTRefFk = relPTaxon.PTRefFk1 " +
238
              	" INNER JOIN PTaxon AS toTaxon ON RelPTaxon.PTNameFk2 = ToTaxon.PTNameFk AND RelPTaxon.PTRefFk2 = ToTaxon.PTRefFk " +
239
              	" INNER JOIN RelPTQualifier q ON q.RelPTQualifierId = RelPTaxon.RelQualifierFk " +
240
            " WHERE RelPTaxon.RelPTaxonId IN ("+ID_LIST_TOKEN+") " +
241
            " ORDER BY RelPTaxon.RelPTaxonId ";
242
		return strQuery;
243
	}
244

    
245
	@Override
246
	public boolean doPartition(ResultSetPartitioner partitioner, BerlinModelImportState state) {
247
		boolean success = true ;
248
		Set<TaxonBase> taxaToSave = new HashSet<>();
249
		Map<String, TaxonBase> taxonMap = partitioner.getObjectMap(BerlinModelTaxonImport.NAMESPACE);
250
		Map<Integer, Classification> classificationMap = new HashMap<>();
251
		Map<String, Reference> refMap = partitioner.getObjectMap(BerlinModelReferenceImport.REFERENCE_NAMESPACE);
252

    
253
		ResultSet rs = partitioner.getResultSet();
254

    
255
		try{
256
			int i = 0;
257
			//for each reference
258
			while (rs.next()){
259

    
260
				if ((i++ % modCount) == 0 && i!= 1 ){ logger.info("RelPTaxa handled: " + (i-1));}
261

    
262
				int relPTaxonId = rs.getInt("RelPTaxonId");
263
				Integer taxon1Id = nullSafeInt(rs, "taxon1Id");
264
				Integer taxon2Id = nullSafeInt(rs, "taxon2Id");
265
				Integer ptRefFk1 = nullSafeInt(rs, "PTRefFk1");
266

    
267
				int relQualifierFk = -1;
268
				try {
269
					Integer relRefFk = nullSafeInt(rs,"relRefFk");
270
					int treeRefFk = rs.getInt("treeRefFk");
271
					int fromRefFk = rs.getInt("fromRefFk");
272

    
273
					relQualifierFk = rs.getInt("relQualifierFk");
274
					String notes = rs.getString("notes");
275
					boolean isConceptRelationship = rs.getBoolean("is_concept_relation");
276

    
277
					TaxonBase<?> taxon1 = taxonMap.get(String.valueOf(taxon1Id));
278
					TaxonBase<?> taxon2 = taxonMap.get(String.valueOf(taxon2Id));
279

    
280
					String refFk = String.valueOf(relRefFk);
281
					Reference citation = refMap.get(refFk);
282

    
283
					String microcitation = null; //does not exist in RelPTaxon
284

    
285
					if (taxon2 != null && taxon1 != null){
286
						if (!(taxon2 instanceof Taxon)){
287
							logger.error("ToTaxon (ID = " + taxon2.getId()+ ", RIdentifier = " + taxon2Id + ") can't be casted to Taxon. RelPTaxon: " + relPTaxonId );
288
							success = false;
289
							continue;
290
						}
291
						AnnotatableEntity taxonRelationship = null;
292
						Taxon toTaxon = (Taxon)taxon2;
293
						if (isTaxonRelationship(relQualifierFk)){
294
							if (!(taxon1 instanceof Taxon)){
295
								logger.error("TaxonBase (ID = " + taxon1.getId()+ ", RIdentifier = " + taxon1Id + ") for TaxonRelation ("+relPTaxonId+") can't be casted to Taxon");
296
								success = false;
297
								continue;
298
							}
299
							Taxon fromTaxon = (Taxon)taxon1;
300
							if (relQualifierFk == TAX_REL_IS_INCLUDED_IN){
301
								taxonRelationship = makeTaxonomicallyIncluded(state, classificationMap, treeRefFk, fromTaxon, toTaxon, citation, microcitation);
302
							}else if (relQualifierFk == TAX_REL_IS_MISAPPLIED_NAME_OF){
303
								boolean isProParte = "p.p.".equals(notes);
304
							    if (isProParte){
305
								    notes = null;
306
								}
307
							    boolean isDoubtful = "?".equals(notes);
308
                                if (isDoubtful){
309
                                    notes = null;
310
                                }
311

    
312
                                if (notes!= null && notes.startsWith("{non ") && notes.endsWith("}")){
313
                                    notes = notes.substring(1, notes.length() - 1);
314
                                }
315
							    //handle auct. author
316
						        if (fromTaxon.getSec() == null || fromTaxon.getSec().getTitleCache().startsWith("auct.")){
317
							        String existingSecTitle = fromTaxon.getSec() == null ? null : fromTaxon.getSec().getTitleCache().trim();
318
						            String existingAppendedPhrase = fromTaxon.getAppendedPhrase();
319
							        if (fromTaxon.getSec() == null && isBlank(existingAppendedPhrase)){
320
							            existingAppendedPhrase = "auct.";
321
							        }
322
						            fromTaxon.setSec(null);
323
							        if (isNotBlank(existingAppendedPhrase) && isNotBlank(notes)){
324
							            logger.warn("Misapplied name has >1 MA relation with a note, RelId: " + relPTaxonId);
325
							        }
326

    
327
							        String newAppendedPhrase = CdmUtils.concat(", ", existingSecTitle, notes);
328
							        fromTaxon.setAppendedPhrase(CdmUtils.concat("; ", existingAppendedPhrase, newAppendedPhrase));
329
							        if (isBlank(fromTaxon.getAppendedPhrase())){
330
							            logger.warn("Appended phrase is empty. This is probably not correct. RelID: " + relPTaxonId);
331
							        }else if ("auct.".equals(fromTaxon.getAppendedPhrase())){
332
							            fromTaxon.setAppendedPhrase(null);
333
							        }
334
							        notes = null;
335
							    }else if (notes != null && notes.startsWith("non ")){
336
							        fromTaxon.setAppendedPhrase(CdmUtils.concat(", ", fromTaxon.getAppendedPhrase(), notes));
337
							        notes = null;
338
							    }
339

    
340
							    if (isProParte){
341
							        taxonRelationship = toTaxon.addProParteMisappliedName(fromTaxon, citation, microcitation);
342
							    }else{
343
							        taxonRelationship = toTaxon.addMisappliedName(fromTaxon, citation, microcitation);
344
	                            }
345
							    if (isDoubtful){
346
							        ((TaxonRelationship)taxonRelationship).setDoubtful(true);
347
							    }
348
                            }else if (relQualifierFk == TAX_REL_IS_PROPARTE_SYN_OF ||
349
                                    //TODO homo/hetero
350
                                    relQualifierFk == TAX_REL_IS_PROPARTE_HOMOTYPIC_SYNONYM_OF ||
351
                                    relQualifierFk == TAX_REL_IS_PROPARTE_HETEROTYPIC_SYNONYM_OF ){
352
                                toTaxon.addProparteSynonym(fromTaxon, citation, microcitation);
353
                            }else if(relQualifierFk == TAX_REL_IS_PARTIAL_SYN_OF ||
354
                                    //TODO homo/hetero
355
                                    relQualifierFk == TAX_REL_IS_PARTIAL_HOMOTYPIC_SYNONYM_OF ||
356
                                    relQualifierFk == TAX_REL_IS_PARTIAL_HETEROTYPIC_SYNONYM_OF ){
357
                                toTaxon.addPartialSynonym(fromTaxon, citation, microcitation);
358
                            }else{
359
								handleAllRelatedTaxa(state, fromTaxon, classificationMap, fromRefFk);
360
								handleAllRelatedTaxa(state, toTaxon, classificationMap, treeRefFk);
361
								logger.warn("Unhandled taxon relationship: RelId:" + relPTaxonId + "; QualifierId: " + relQualifierFk);
362
							}
363
						}else if (isSynonymRelationship(relQualifierFk)){
364
							if (!(taxon1 instanceof Synonym)){
365
								logger.warn("Validated: Taxon (ID = " + taxon1.getId()+ ", RIdentifier = " + taxon1Id + ") can't be casted to Synonym");
366
								success = false;
367
								continue;
368
							}
369
							handleAllRelatedTaxa(state, toTaxon, classificationMap, treeRefFk);
370
							Synonym synonym = (Synonym)taxon1;
371
							if (synonym.getAcceptedTaxon()!= null){
372
							    logger.warn("RelID: " + relPTaxonId + ". Synonym ("+taxon1Id +") already has an accepted taxon. Create clone.");
373
							    synonym = (Synonym)synonym.clone();
374
							}
375
							makeSynRel(state, relQualifierFk, toTaxon, synonym, citation, microcitation);
376

    
377
							if (relQualifierFk == TAX_REL_IS_SYNONYM_OF ||
378
									relQualifierFk == TAX_REL_IS_HOMOTYPIC_SYNONYM_OF ||
379
									relQualifierFk == TAX_REL_IS_HETEROTYPIC_SYNONYM_OF){
380
							}else{
381
								success = false;
382
								logger.warn("Synonym relationship type not yet implemented: " + relQualifierFk);
383
							}
384
							//
385
							notes = handleSynonymNotes(state, toTaxon, synonym, notes, relPTaxonId);
386
						}else if (isConceptRelationship){
387
							ResultWrapper<Boolean> isInverse = ResultWrapper.NewInstance(false);
388
							ResultWrapper<Boolean> isDoubtful = ResultWrapper.NewInstance(false);
389
							try {
390
								TaxonRelationshipType relType = BerlinModelTransformer.taxonRelId2TaxonRelType(relQualifierFk, isInverse, isDoubtful);
391

    
392
								if (! (taxon1 instanceof Taxon)){
393
									success = false;
394
									logger.error("TaxonBase (ID = " + taxon1.getId()+ ", RIdentifier = " + taxon1Id + ") can't be casted to Taxon");
395
								}else{
396
									Taxon fromTaxon = (Taxon)taxon1;
397
									if (isInverse.getValue() == true){
398
										Taxon tmp = fromTaxon;
399
										fromTaxon = toTaxon;
400
										toTaxon = tmp;
401
									}
402
									taxonRelationship = fromTaxon.addTaxonRelation(toTaxon, relType, citation, microcitation);
403
									handleAllRelatedTaxa(state, toTaxon, classificationMap, treeRefFk);
404
									handleAllRelatedTaxa(state, fromTaxon, classificationMap, fromRefFk);
405
									if (isDoubtful.getValue() == true){
406
										((TaxonRelationship)taxonRelationship).setDoubtful(true);
407
									}
408
								}
409
							} catch (UnknownCdmTypeException e) {
410
								logger.warn("TaxonRelationShipType " + relQualifierFk + " (conceptRelationship) not yet implemented");
411
								 success = false;
412
							}
413
						}else {
414
							logger.warn("TaxonRelationShipType " + relQualifierFk + " not yet implemented: RelPTaxonId = " + relPTaxonId );
415
							success = false;
416
						}
417

    
418
						if (taxonRelationship != null && isNotBlank(notes)){
419
						    doNotes(taxonRelationship, notes);
420
						}
421
						if (isNotBlank(notes)){
422
						    logger.warn("Notes in RelPTaxon should all be handled explicitly and should not exist as notes anymore. RelID: " + relPTaxonId + ". Note: " + notes);
423
						}
424
						taxaToSave.add(taxon2);
425

    
426
						//TODO
427
						//etc.
428
					}else{
429
						if (taxon2 != null && taxon1 == null){
430
							logger.warn("First taxon ("+taxon1Id+") for RelPTaxon " + relPTaxonId + " does not exist in store. RelType: " + relQualifierFk);
431
						}else if (taxon2 == null && taxon1 != null){
432
							logger.warn("Second taxon ("+taxon2Id +") for RelPTaxon " + relPTaxonId + " does not exist in store. RelType: " + relQualifierFk);
433
						}else{
434
							logger.warn("Both taxa ("+taxon1Id+","+taxon2Id +") for RelPTaxon " + relPTaxonId + " do not exist in store. RelType: " + relQualifierFk);
435
						}
436

    
437
						success = false;
438
					}
439
				} catch (Exception e) {
440
					logger.error("Exception occurred when trying to handle taxon relationship " + relPTaxonId + " relQualifierFK " + relQualifierFk + " (" + taxon1Id + ","+ taxon2Id + "): " + e.getMessage());
441
					e.printStackTrace();
442
				}
443
			}
444
		}catch(SQLException e){
445
			throw new RuntimeException(e);
446
		}
447
		logger.info("Taxa to save: " + taxaToSave.size());
448
		partitioner.startDoSave();
449
		getTaxonService().saveOrUpdate(taxaToSave);
450
		classificationMap = null;
451
		taxaToSave = null;
452

    
453
		return success;
454
	}
455

    
456

    
457
	/**
458
     * @param toTaxon
459
     * @param synonym
460
     * @param notes
461
     * @return
462
     */
463
    private String handleSynonymNotes(BerlinModelImportState state, Taxon toTaxon, Synonym synonym, String notes, int relId) {
464
        if (state.getConfig().isEuroMed() && isNotBlank(notes)){
465
            notes = notes.trim();
466
            if (notes.startsWith("[non ") && notes.endsWith("]")){
467
                notes = notes.substring(5, notes.length()-1).trim();
468
                String[] splits = notes.split(", nec ");
469
                for (String split : splits){
470
                    String nameStr = split.replace("<i>", "").replace("</i>", "");
471
                    NonViralNameParserImpl parser = NonViralNameParserImpl.NewInstance();
472
                    TaxonName name;
473
                    NomenclaturalStatusType status = null;
474
                    if (nameStr.endsWith(", nom. rej.") || nameStr.endsWith(", nom. cons.")||nameStr.endsWith(", nom. illeg.")){
475
                        String statusStr = nameStr.endsWith(", nom. rej.")? ", nom. rej.":
476
                            nameStr.endsWith(", nom. cons.")? ", nom. cons.":
477
                            ", nom. illeg.";
478
                        nameStr = nameStr.replace(statusStr, "");
479
                        statusStr = statusStr.replace(", ", "");
480
                        try {
481
                            status = NomenclaturalStatusType.getNomenclaturalStatusTypeByAbbreviation(statusStr, null);
482
                        } catch (UnknownCdmTypeException e) {
483
                            logger.warn("NomStatusType not recognized: "+  statusStr + ", RelId: " +  relId);
484
                        }
485
                    }
486

    
487
                    if (nameStr.contains(",") || nameStr.contains(" in ") ){
488
                        name = parser.parseReferencedName(nameStr, state.getConfig().getNomenclaturalCode(), null);
489
                    }else if (nameStr.matches(".*\\s\\d{4}")){
490
                        String nameStr2 = nameStr.substring(0, nameStr.length() - 5).trim();
491
                        String yearStr = nameStr.substring(nameStr.length()-4);
492
                        name = (TaxonName)parser.parseFullName(nameStr2, state.getConfig().getNomenclaturalCode(), null);
493
                        Reference nomRef = name.getNomenclaturalReference();
494
                        if (nomRef == null){
495
                            nomRef = ReferenceFactory.newGeneric();
496
                            name.setNomenclaturalReference(nomRef);
497
                        }
498
                        nomRef.setDatePublished(TimePeriodParser.parseStringVerbatim(yearStr));
499
                    }else if (nameStr.endsWith(" 1831-1832")){
500
                        String nameStr2 = nameStr.substring(0, nameStr.length() - 10).trim();
501
                        name = (TaxonName)parser.parseFullName(nameStr2, state.getConfig().getNomenclaturalCode(), null);
502
                        Reference nomRef = name.getNomenclaturalReference();
503
                        if (nomRef == null){
504
                            nomRef = ReferenceFactory.newGeneric();
505
                            name.setNomenclaturalReference(nomRef);
506
                        }
507
                        nomRef.setDatePublished(TimePeriodParser.parseStringVerbatim("1831-1832"));
508
                    }else{
509
                        name = parser.parseReferencedName(nameStr, state.getConfig().getNomenclaturalCode(), null);
510
                    }
511
                    if (name.isProtectedTitleCache() || name.isProtectedNameCache()
512
                            || name.getNomenclaturalReference() != null && (name.getNomenclaturalReference().isProtectedAbbrevTitleCache()|| name.getNomenclaturalReference().isProtectedTitleCache() )){
513
                        logger.warn("Blocking name for synonym relation could not be parsed: " + nameStr + ", RelId: "+ relId);
514
                    }
515
                    if (status != null){
516
                        name.addStatus(NomenclaturalStatus.NewInstance(status));
517
                    }
518
                    synonym.getName().addRelationshipFromName(name, NameRelationshipType.BLOCKING_NAME_FOR(), null, null, null);
519

    
520
                    getNameService().saveOrUpdate(name);
521
                }
522
                return null;
523
            }else{
524
                return notes;
525
            }
526
        }else{
527
            return notes;
528
        }
529
    }
530

    
531
    private void handleAllRelatedTaxa(BerlinModelImportState state, Taxon taxon,
532
            Map<Integer, Classification> classificationMap, Integer secRefFk) {
533
		if (taxon.getTaxonNodes().size() > 0){
534
			return;
535
		}else{
536
			Classification classification = getClassificationTree(state, classificationMap, secRefFk);
537
			classification.addChildTaxon(taxon, null, null);
538
		}
539
	}
540

    
541
	@Override
542
	protected void doInvoke(BerlinModelImportState state){
543
		try {
544
			makeClassifications(state);
545
			super.doInvoke(state);
546
			makeFlatClassificationTaxa(state);
547
			return;
548
		} catch (SQLException e) {
549
			throw new RuntimeException(e);
550
		}
551

    
552
	}
553

    
554

    
555
	private void makeFlatClassificationTaxa(BerlinModelImportState state) {
556
		//Note: this part still does not use partitions
557
		logger.info("Flat classifications start");
558
		TransactionStatus txStatus = startTransaction();
559
		if (! state.getConfig().isIncludeFlatClassifications()){
560
			return;
561
		}
562
		String sql = " SELECT pt.PTRefFk AS secRefFk, pt.RIdentifier " +
563
						" FROM PTaxon AS pt " +
564
							" LEFT OUTER JOIN RelPTaxon ON pt.PTNameFk = RelPTaxon.PTNameFk2 AND pt.PTRefFk = RelPTaxon.PTRefFk2 " +
565
							"  LEFT OUTER JOIN RelPTaxon AS RelPTaxon_1 ON pt.PTNameFk = RelPTaxon_1.PTNameFk1 AND pt.PTRefFk = RelPTaxon_1.PTRefFk1 " +
566
						" WHERE (RelPTaxon_1.RelQualifierFk IS NULL) AND (dbo.RelPTaxon.RelQualifierFk IS NULL) " +
567
						" ORDER BY pt.PTRefFk "	;
568
		ResultSet rs = state.getConfig().getSource().getResultSet(sql);
569
		Map<Object, Map<String, ? extends CdmBase>> maps = getRelatedObjectsForFlatPartition(rs);
570

    
571
		Map<String, TaxonBase> taxonMap = (Map<String, TaxonBase>) maps.get(BerlinModelTaxonImport.NAMESPACE);
572
		Map<Integer, Classification> classificationMap = new HashMap<>();
573

    
574
		rs = state.getConfig().getSource().getResultSet(sql);
575
		try {
576
			while (rs.next()){
577
				Integer treeRefFk = rs.getInt("secRefFk");
578
				String taxonId = rs.getString("RIdentifier");
579
				Classification classification = getClassificationTree(state, classificationMap, treeRefFk);
580
				TaxonBase<?> taxon = taxonMap.get(taxonId);
581
				if (taxon == null){
582
					String message = "TaxonBase for taxon id (%s) not found in taxonMap";
583
					logger.warn(String.format(message, taxonId, taxonId));
584
				}else if (taxon.isInstanceOf(Taxon.class)){
585
					classification.addChildTaxon(CdmBase.deproxy(taxon, Taxon.class), null, null);
586
				}else{
587
					String message = "TaxonBase for taxon is not of class Taxon but %s (RIdentifier %s)";
588
					logger.warn(String.format(message, taxon.getClass(), taxonId));
589
				}
590
			}
591
		} catch (SQLException e) {
592
			e.printStackTrace();
593
		}
594
		commitTransaction(txStatus);
595
		logger.info("Flat classifications end");
596

    
597
	}
598

    
599
	@Override
600
	protected String getIdQuery(BerlinModelImportState state) {
601
		if (state.getConfig().getRelTaxaIdQuery() != null){
602
			return state.getConfig().getRelTaxaIdQuery();
603
		}else{
604
			return super.getIdQuery(state);
605
		}
606
	}
607

    
608
	@Override
609
	public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs, BerlinModelImportState state) {
610
		String nameSpace;
611
		Class<?> cdmClass;
612
		Set<String> idSet;
613
		Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<>();
614

    
615
		try{
616
			Set<String> taxonIdSet = new HashSet<>();
617
			Set<String> referenceIdSet = new HashSet<>();
618
//			Set<String> classificationIdSet = new HashSet<>();
619
			while (rs.next()){
620
				handleForeignKey(rs, taxonIdSet, "taxon1Id");
621
				handleForeignKey(rs, taxonIdSet, "taxon2Id");
622
//				handleForeignKey(rs, classificationIdSet, "treeRefFk");
623
				handleForeignKey(rs, referenceIdSet, "RelRefFk");
624
	}
625

    
626
			//taxon map
627
			nameSpace = BerlinModelTaxonImport.NAMESPACE;
628
			cdmClass = TaxonBase.class;
629
			idSet = taxonIdSet;
630
			Map<String, TaxonBase> taxonMap = (Map<String, TaxonBase>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
631
			result.put(nameSpace, taxonMap);
632

    
633
//			//tree map
634
//			nameSpace = "Classification";
635
//			cdmClass = Classification.class;
636
//			idSet = classificationIdSet;
637
//			Map<String, Classification> treeMap = (Map<String, Classification>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
638
//			result.put(cdmClass, treeMap);
639
//			Set<UUID> treeUuidSet = state
640
//			getClassificationService().find(uuidSet);
641
//
642
			//reference map
643
			nameSpace = BerlinModelReferenceImport.REFERENCE_NAMESPACE;
644
			cdmClass = Reference.class;
645
			idSet = referenceIdSet;
646
			Map<String, Reference> referenceMap = (Map<String, Reference>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
647
			result.put(nameSpace, referenceMap);
648

    
649
		} catch (SQLException e) {
650
			throw new RuntimeException(e);
651
		}
652
		return result;
653
	}
654

    
655

    
656
	private Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForFlatPartition( ResultSet rs) {
657
		String nameSpace;
658
		Class cdmClass;
659
		Set<String> idSet;
660
		Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<>();
661

    
662
		try{
663
			Set<String> taxonIdSet = new HashSet<>();
664
			Set<String> referenceIdSet = new HashSet<>();
665
//			Set<String> classificationIdSet = new HashSet<>();
666
			while (rs.next()){
667
				handleForeignKey(rs, taxonIdSet, "RIdentifier");
668
//				handleForeignKey(rs, classificationIdSet, "treeRefFk");
669
			}
670

    
671
			//taxon map
672
			nameSpace = BerlinModelTaxonImport.NAMESPACE;
673
			cdmClass = TaxonBase.class;
674
			idSet = taxonIdSet;
675
			Map<String, TaxonBase> taxonMap = (Map<String, TaxonBase>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
676
			result.put(nameSpace, taxonMap);
677

    
678
//			//tree map
679
//			nameSpace = "Classification";
680
//			cdmClass = Classification.class;
681
//			idSet = classificationIdSet;
682
//			Map<String, Classification> treeMap = (Map<String, Classification>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
683
//			result.put(cdmClass, treeMap);
684
//			Set<UUID> treeUuidSet = state
685
//			getClassificationService().find(uuidSet);
686
//
687

    
688
		} catch (SQLException e) {
689
			throw new RuntimeException(e);
690
		}
691
		return result;
692
	}
693

    
694

    
695
	private void makeSynRel (BerlinModelImportState state, int relQualifierFk, Taxon toTaxon, Synonym synonym, Reference citation, String microcitation){
696
		if (state.getConfig().isWarnForDifferingSynonymReference() && citation != null && !citation.equals(synonym.getSec())){
697
		    logger.warn("A synonym relationship citation is given and differs from synonym secundum. This can not be handled in CDM");
698
		}
699
		if (isNotBlank(microcitation)  && !microcitation.equals(synonym.getSecMicroReference())){
700
            logger.warn("A synonym relationship microcitation is given and differs from synonym secundum micro reference. This can not be handled in CDM");
701
        }
702
	    if (relQualifierFk == TAX_REL_IS_HOMOTYPIC_SYNONYM_OF ||
703
				relQualifierFk == TAX_REL_IS_PROPARTE_HOMOTYPIC_SYNONYM_OF ||
704
				relQualifierFk == TAX_REL_IS_PARTIAL_HOMOTYPIC_SYNONYM_OF){
705
			toTaxon.addHomotypicSynonym(synonym);
706
		}else if (relQualifierFk == TAX_REL_IS_HETEROTYPIC_SYNONYM_OF ||
707
				relQualifierFk == TAX_REL_IS_PROPARTE_HETEROTYPIC_SYNONYM_OF ||
708
				relQualifierFk == TAX_REL_IS_PARTIAL_HETEROTYPIC_SYNONYM_OF){
709
			toTaxon.addSynonym(synonym, SynonymType.HETEROTYPIC_SYNONYM_OF());
710
		}else if (relQualifierFk == TAX_REL_IS_SYNONYM_OF ||
711
				relQualifierFk == TAX_REL_IS_PROPARTE_SYN_OF ||
712
				relQualifierFk == TAX_REL_IS_PARTIAL_SYN_OF){
713
			toTaxon.addSynonym(synonym, SynonymType.SYNONYM_OF());
714
		}else{
715
			logger.warn("SynonymyRelationShipType could not be defined for relQualifierFk " + relQualifierFk + ". 'Unknown'-Type taken instead.");
716
			toTaxon.addSynonym(synonym, SynonymType.SYNONYM_OF());
717
		}
718
		return;
719

    
720
	}
721

    
722
	private  boolean isSynonymRelationship(int relQualifierFk){
723
		if (relQualifierFk == TAX_REL_IS_SYNONYM_OF ||
724
			relQualifierFk == TAX_REL_IS_HOMOTYPIC_SYNONYM_OF ||
725
			relQualifierFk == TAX_REL_IS_HETEROTYPIC_SYNONYM_OF ||
726
			relQualifierFk == TAX_REL_IS_PROPARTE_SYN_OF ||
727
			relQualifierFk == TAX_REL_IS_PARTIAL_SYN_OF ||
728
			relQualifierFk == TAX_REL_IS_PROPARTE_HOMOTYPIC_SYNONYM_OF ||
729
			relQualifierFk == TAX_REL_IS_PROPARTE_HETEROTYPIC_SYNONYM_OF ||
730
			relQualifierFk == TAX_REL_IS_PARTIAL_HOMOTYPIC_SYNONYM_OF ||
731
			relQualifierFk == TAX_REL_IS_PARTIAL_HETEROTYPIC_SYNONYM_OF
732
		){
733
			return true;
734
		}else{
735
			return false;
736
		}
737
	}
738

    
739
	private  boolean isTaxonRelationship(int relQualifierFk){
740
		if (relQualifierFk == TAX_REL_IS_INCLUDED_IN ||
741
				relQualifierFk == TAX_REL_IS_MISAPPLIED_NAME_OF){
742
			return true;
743
		}else if (relQualifierFk == TAX_REL_IS_PROPARTE_SYN_OF ||
744
	                relQualifierFk == TAX_REL_IS_PARTIAL_SYN_OF ||
745
	                relQualifierFk == TAX_REL_IS_PROPARTE_HOMOTYPIC_SYNONYM_OF ||
746
	                relQualifierFk == TAX_REL_IS_PROPARTE_HETEROTYPIC_SYNONYM_OF ||
747
	                relQualifierFk == TAX_REL_IS_PARTIAL_HOMOTYPIC_SYNONYM_OF ||
748
	                relQualifierFk == TAX_REL_IS_PARTIAL_HETEROTYPIC_SYNONYM_OF
749
                    ){
750
	            return true;
751

    
752
		}else{
753
			return false;
754
		}
755
	}
756

    
757
	private TaxonNode makeTaxonomicallyIncluded(BerlinModelImportState state, Map<Integer, Classification> classificationMap, int treeRefFk, Taxon child, Taxon parent, Reference citation, String microCitation){
758
		Classification tree = getClassificationTree(state, classificationMap, treeRefFk);
759
		return tree.addParentChild(parent, child, citation, microCitation);
760
	}
761

    
762
	private Classification getClassificationTree(BerlinModelImportState state, Map<Integer, Classification> classificationMap, int treeRefFk) {
763
		if (state.getConfig().isUseSingleClassification()){
764
			if (state.getConfig().getSourceSecId() != null){
765
				treeRefFk = (Integer)state.getConfig().getSourceSecId();
766
			}else{
767
				treeRefFk = 1;
768
			}
769

    
770
		}
771
		Classification tree = classificationMap.get(treeRefFk);
772
		if (tree == null){
773
			UUID treeUuid = state.getTreeUuidByIntTreeKey(treeRefFk);
774
			if (treeUuid == null){
775
				throw new IllegalStateException("treeUUID does not exist in state for " + treeRefFk );
776
			}
777
			tree = getClassificationService().find(treeUuid);
778
			classificationMap.put(treeRefFk, tree);
779
		}
780
		if (tree == null){
781
			throw new IllegalStateException("Tree for ToTaxon reference " + treeRefFk + " does not exist.");
782
		}
783
		return tree;
784
	}
785

    
786
	@Override
787
	protected boolean doCheck(BerlinModelImportState state){
788
		IOValidator<BerlinModelImportState> validator = new BerlinModelTaxonRelationImportValidator();
789
		return validator.validate(state);
790
	}
791

    
792
	@Override
793
	protected boolean isIgnore(BerlinModelImportState state){
794
		return ! state.getConfig().isDoRelTaxa();
795
	}
796

    
797

    
798
}
(18-18/22)