Project

General

Profile

Download (49 KB) Statistics
| Branch: | Revision:
1
/**
2
* Copyright (C) 2007 EDIT
3
* European Distributed Institute of Taxonomy
4
* http://www.e-taxonomy.eu
5
*
6
* The contents of this file are subject to the Mozilla Public License Version 1.1
7
* See LICENSE.TXT at the top of this package for the full license terms.
8
*/
9

    
10
package eu.etaxonomy.cdm.io.pesi.faunaEuropaea;
11

    
12
import static eu.etaxonomy.cdm.io.pesi.faunaEuropaea.FaunaEuropaeaTransformer.A_AUCT;
13

    
14
import java.sql.ResultSet;
15
import java.sql.SQLException;
16
import java.util.ArrayList;
17
import java.util.HashMap;
18
import java.util.HashSet;
19
import java.util.List;
20
import java.util.Map;
21
import java.util.Set;
22
import java.util.UUID;
23

    
24
import org.apache.logging.log4j.LogManager;
25
import org.apache.logging.log4j.Logger;
26
import org.hibernate.Query;
27
import org.springframework.stereotype.Component;
28
import org.springframework.transaction.TransactionStatus;
29

    
30
import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
31
import eu.etaxonomy.cdm.io.common.ICdmIO;
32
import eu.etaxonomy.cdm.io.common.MapWrapper;
33
import eu.etaxonomy.cdm.io.common.Source;
34
import eu.etaxonomy.cdm.io.pesi.out.PesiTransformer;
35
import eu.etaxonomy.cdm.model.agent.AgentBase;
36
import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;
37
import eu.etaxonomy.cdm.model.common.CdmBase;
38
import eu.etaxonomy.cdm.model.common.IdentifiableSource;
39
import eu.etaxonomy.cdm.model.common.Marker;
40
import eu.etaxonomy.cdm.model.common.MarkerType;
41
import eu.etaxonomy.cdm.model.name.Rank;
42
import eu.etaxonomy.cdm.model.name.TaxonName;
43
import eu.etaxonomy.cdm.model.reference.Reference;
44
import eu.etaxonomy.cdm.model.taxon.Classification;
45
import eu.etaxonomy.cdm.model.taxon.Synonym;
46
import eu.etaxonomy.cdm.model.taxon.SynonymType;
47
import eu.etaxonomy.cdm.model.taxon.Taxon;
48
import eu.etaxonomy.cdm.model.taxon.TaxonBase;
49
import eu.etaxonomy.cdm.model.taxon.TaxonNode;
50
//import eu.etaxonomy.cdm.profiler.ProfilerController;
51
import eu.etaxonomy.cdm.model.taxon.TaxonNodeAgentRelation;
52

    
53

    
54

    
55
/**
56
 * @author a.babadshanjan
57
 * @since 12.05.2009
58
 */
59
@Component
60
public class FaunaEuropaeaRelTaxonIncludeImport extends FaunaEuropaeaImportBase  {
61

    
62
    private static final long serialVersionUID = 4068337179814111891L;
63
    private static Logger logger = LogManager.getLogger();
64

    
65
    public static final String OS_NAMESPACE_TAXON = "Taxon";
66
	private static final String AUCT_STRING = "auct.";
67
	//private static final String acceptedTaxonUUID = "A9C24E42-69F5-4681-9399-041E652CF338"; // any accepted taxon uuid, taken from original fauna europaea database
68
	//private static final String acceptedTaxonUUID = "E23E6295-836A-4332-BF72-7D29949C7C60"; //faunaEu_1_3
69
	//private static final String acceptedTaxonUUID = "bab7642e-f733-4a21-848d-a15250d2f4ed"; //for faunEu (2.4)
70
	private static final String acceptedTaxonUUID = "DADA6F44-B7B5-4C0A-9F32-980F54B02C36"; // for MfNFaunaEuropaea
71

    
72
	private Map<UUID, TeamOrPersonBase> agentMap = new HashMap<>();
73

    
74

    
75
	private Reference sourceRef;
76
    private final String parentPluralString = "Taxa";
77
	private static String ALL_SYNONYM_FROM_CLAUSE = " FROM Taxon INNER JOIN Taxon AS Parent " +
78
	" ON Taxon.TAX_TAX_IDPARENT = Parent.TAX_ID " +
79
	" WHERE (Taxon.TAX_VALID = 0) " +
80
	" AND (Taxon.TAX_AUT_ID <> " + A_AUCT + " OR Taxon.TAX_AUT_ID IS NULL)";
81

    
82
	@Override
83
	protected boolean doCheck(FaunaEuropaeaImportState state) {
84
		boolean result = true;
85
		FaunaEuropaeaImportConfigurator fauEuConfig = state.getConfig();
86
		logger.warn("Checking for Taxa not yet fully implemented");
87
		result &= checkTaxonStatus(fauEuConfig);
88

    
89
		return result;
90
	}
91

    
92
	@Override
93
	protected boolean isIgnore(FaunaEuropaeaImportState state) {
94
		return ! (state.getConfig().isDoTaxonomicallyIncluded() ||
95
		state.getConfig().isDoMisappliedNames() || state.getConfig().isDoHeterotypicSynonyms() || state.getConfig().isDoInferredSynonyms());
96
	}
97

    
98
	private boolean checkTaxonStatus(FaunaEuropaeaImportConfigurator fauEuConfig) {
99
		boolean result = true;
100
//		try {
101
			Source source = fauEuConfig.getSource();
102
			String sqlStr = "";
103
			ResultSet rs = source.getResultSet(sqlStr);
104
			return result;
105
//		} catch (SQLException e) {
106
//			e.printStackTrace();
107
//			return false;
108
//		}
109
	}
110

    
111
	@Override
112
	protected void doInvoke(FaunaEuropaeaImportState state) {
113

    
114
		/*logger.warn("Start RelTaxon doInvoke");
115
		ProfilerController.memorySnapshot();
116
		*/
117
	    TransactionStatus txStatus = startTransaction();
118
	    if (state.getAgentMap().isEmpty()){
119
	        createAgentMap(state);
120
	    }
121
		Map<String, MapWrapper<? extends CdmBase>> stores = state.getStores();
122

    
123
		MapWrapper<TeamOrPersonBase> authorStore = (MapWrapper<TeamOrPersonBase>)stores.get(ICdmIO.TEAM_STORE);
124
		authorStore.makeEmpty();
125

    
126
		if(logger.isInfoEnabled()) { logger.info("Start making relationships..."); }
127

    
128

    
129

    
130
		// the uuid of an accepted taxon is needed here. any accepted taxon will do.
131
		TaxonBase<?> taxon = getTaxonService().find(UUID.fromString(acceptedTaxonUUID));
132
		sourceRef = taxon.getSec();
133

    
134
		Classification tree = getClassificationFor(state, sourceRef);
135
		commitTransaction(txStatus);
136

    
137
		logger.warn("Before processParentsChildren " + state.getConfig().isDoTaxonomicallyIncluded());
138

    
139
		//ProfilerController.memorySnapshot();
140

    
141
		if (state.getConfig().isDoTaxonomicallyIncluded())  {
142
			processParentsChildren(state);
143

    
144
		}
145
		if (state.getConfig().isDoAssociatedSpecialists()){
146
			processAssociatedSpecialists(state);
147
		}
148

    
149
		logger.warn("Before processMissappliedNames " + state.getConfig().isDoMisappliedNames());
150

    
151
		//ProfilerController.memorySnapshot();
152

    
153
		if (state.getConfig().isDoMisappliedNames()) {
154
			processMisappliedNames(state);
155
		}
156
		/*
157
		logger.warn("Before heterotypic synonyms");
158
		ProfilerController.memorySnapshot();
159
		*/
160
		if (state.getConfig().isDoHeterotypicSynonyms()) {
161
			if(logger.isInfoEnabled()) {
162
				logger.info("Start making heterotypic synonyms ...");
163
			}
164
			processHeterotypicSynonyms(state, ALL_SYNONYM_FROM_CLAUSE);
165
		}
166

    
167
		if (state.getConfig().isDoInferredSynonyms()){
168
		    processInferredSynonyms(state);
169
		}
170
		/*
171
		logger.warn("End RelTaxon doInvoke");
172
		ProfilerController.memorySnapshot();
173
		*/
174
		logger.info("End making relationships......");
175

    
176
		return;
177
	}
178

    
179
	/**
180
     * @param state
181
     */
182
    private void createAgentMap(FaunaEuropaeaImportState state) {
183

    
184
        List<String> propertyPaths = new ArrayList<String>();
185
        propertyPaths.add("sources");
186

    
187
        Query query =  getAgentService().getSession().createQuery("select ab.uuid, ob.idInSource from AgentBase ab join ab.sources ob where ob.idNamespace like 'User' ");
188
        List<Object[]> result = query.list();
189
       // List<TeamOrPersonBase> agents = getAgentService().list(TeamOrPersonBase.class, 1000, 0, null, null);
190

    
191
        Integer idInSource;
192
        for (Object[] person: result){
193
            idInSource = Integer.valueOf((String)person[1]);
194
            UUID agentUuid = (UUID) person[0];
195
            state.getAgentMap().put(idInSource, agentUuid);
196
        }
197
    }
198

    
199
    /** Retrieve child-parent uuid map from CDM DB */
200
	private void processParentsChildren(FaunaEuropaeaImportState state) {
201

    
202
		int limit = state.getConfig().getLimitSave();
203

    
204
		TransactionStatus txStatus = null;
205

    
206
		Map<UUID, UUID> childParentMap = null;
207

    
208
		Map<UUID, UUID> taxonSpecialistMap = null;
209
		Map<UUID, UUID> taxonGroupCoordinatorMap = null;
210
		FaunaEuropaeaImportConfigurator fauEuConfig = state.getConfig();
211
		Source source = fauEuConfig.getSource();
212
		int i = 0;
213

    
214
		String selectCount =
215
			" SELECT count(*) ";
216

    
217
		String selectColumns = " SELECT Taxon.UUID AS ChildUuid, Taxon.tax_usr_idgc as group_coordinator, Taxon.tax_usr_idsp as specialist, Parent.UUID AS ParentUuid, Parent.tax_usr_idgc as parent_group_coordinator, Parent.tax_usr_idsp as parent_specialist";
218

    
219
		String fromClause = " FROM Taxon INNER JOIN Taxon AS Parent " +
220
		" ON Taxon.TAX_TAX_IDPARENT = Parent.TAX_ID " +
221
		" WHERE (Taxon.TAX_VALID <> 0) AND (Taxon.TAX_AUT_ID <> " + A_AUCT + " OR Taxon.TAX_AUT_ID IS NULL )";
222

    
223
		String orderClause = " ORDER BY Taxon.TAX_RNK_ID ASC";
224

    
225
		String countQuery =
226
			selectCount + fromClause;
227

    
228
		String selectQuery =
229
			selectColumns + fromClause + orderClause;
230

    
231
		if(logger.isInfoEnabled()) { logger.info("Start making taxonomically included relationships..."); }
232

    
233
		try {
234

    
235
			ResultSet rs = source.getResultSet(countQuery);
236
			rs.next();
237
			int count = rs.getInt(1);
238

    
239
			rs = source.getResultSet(selectQuery);
240

    
241
	        if (logger.isInfoEnabled()) {
242
				logger.info("Number of rows: " + count);
243
				logger.info("Count Query: " + countQuery);
244
				logger.info("Select Query: " + selectQuery);
245
			}
246

    
247
	        while (rs.next()) {
248

    
249
				if ((i++ % limit) == 0) {
250

    
251
					txStatus = startTransaction();
252
					childParentMap = new HashMap<UUID, UUID>(limit);
253
					taxonSpecialistMap = new HashMap<UUID,UUID>(limit);
254
					taxonGroupCoordinatorMap = new HashMap<UUID, UUID>(limit);
255
					if(logger.isInfoEnabled()) {
256
						logger.info("Taxonomically included retrieved: " + (i-1));
257
					}
258
				}
259

    
260
				String childUuidStr = rs.getString("ChildUuid");
261
				String parentUuidStr = rs.getString("ParentUuid");
262

    
263
				int group_coordinator_id = rs.getInt("group_coordinator");
264
				int specialist_id = rs.getInt("specialist");
265
				int parent_group_coordinator_id = rs.getInt("parent_group_coordinator");
266
                int parent_specialist_id = rs.getInt("parent_specialist");
267
				UUID childUuid = UUID.fromString(childUuidStr);
268
				UUID parentUuid = UUID.fromString(parentUuidStr);
269

    
270
				if (!childParentMap.containsKey(childUuid)) {
271

    
272
						childParentMap.put(childUuid, parentUuid);
273

    
274

    
275
				} else {
276
					if(logger.isInfoEnabled()) {
277
						logger.info("Duplicated child UUID (" + childUuid + ")");
278
					}
279
				}
280
				if (!taxonSpecialistMap.containsKey(childUuid)) {
281

    
282
				    taxonSpecialistMap.put(childUuid, state.getAgentMap().get(specialist_id));
283
				}
284
				if (!taxonSpecialistMap.containsKey(parentUuid)) {
285

    
286
                    taxonSpecialistMap.put(parentUuid, state.getAgentMap().get(parent_specialist_id));
287
                }
288
				if (!taxonGroupCoordinatorMap.containsKey(childUuid)) {
289

    
290
				    taxonGroupCoordinatorMap.put(childUuid, state.getAgentMap().get(group_coordinator_id));
291
                }
292
                if (!taxonGroupCoordinatorMap.containsKey(parentUuid)) {
293

    
294
                    taxonGroupCoordinatorMap.put(parentUuid, state.getAgentMap().get(parent_group_coordinator_id));
295
                }
296

    
297

    
298
				if (((i % limit) == 0 && i != 1 ) || i == count ) {
299

    
300
					createAndCommitParentChildRelationships(
301
							state, txStatus, childParentMap, taxonSpecialistMap, taxonGroupCoordinatorMap);
302
					childParentMap = null;
303

    
304
					if(logger.isInfoEnabled()) {
305
						logger.info("i = " + i + " - Transaction committed");
306
					}
307
				}
308
			}
309
	        rs = null;
310
	        if (childParentMap != null){
311
	        	logger.info("processParentsChildren... last commit");
312
	        	createAndCommitParentChildRelationships(
313
						state, txStatus, childParentMap,taxonSpecialistMap, taxonGroupCoordinatorMap);
314
	        	childParentMap = null;
315
	        }
316

    
317
		} catch (Exception e) {
318
			logger.error("Exception:" +  e);
319
			state.setUnsuccessfull();
320
		}
321
		agentMap = null;
322
		childParentMap = null;
323
		return;
324
	}
325

    
326
	private Map<UUID, UUID> createAndCommitParentChildRelationships(
327
			FaunaEuropaeaImportState state, TransactionStatus txStatus,
328
			Map<UUID, UUID> childParentMap, Map<UUID, UUID> taxonSpecialistMap, Map<UUID, UUID> taxonGroupCoordinatorMap) {
329
		createParentChildRelationships(state, childParentMap, taxonSpecialistMap, taxonGroupCoordinatorMap, txStatus);
330

    
331

    
332
		commitTransaction(txStatus);
333
		return childParentMap;
334
	}
335

    
336

    
337
	/** Retrieve misapplied name / accepted taxon uuid map from CDM DB */
338
	private void processMisappliedNames(FaunaEuropaeaImportState state) {
339

    
340
		int limit = state.getConfig().getLimitSave();
341

    
342
		TransactionStatus txStatus = null;
343

    
344
		Map<UUID, UUID> childParentMap = null;
345
		FaunaEuropaeaImportConfigurator fauEuConfig = state.getConfig();
346
		Source source = fauEuConfig.getSource();
347
		int i = 0;
348

    
349
		String selectCount =
350
			" SELECT count(*) ";
351

    
352
		String selectColumns = " SELECT Taxon.UUID AS MisappliedUuid, Parent.UUID AS AcceptedUuid ";
353

    
354
		String fromClause = " FROM Taxon INNER JOIN Taxon AS Parent " +
355
		" ON Taxon.TAX_TAX_IDPARENT = Parent.TAX_ID " +
356
		" WHERE (Taxon.TAX_VALID = 0) AND (Taxon.TAX_AUT_ID = " + A_AUCT + ")";
357
		String orderClause = " ORDER BY dbo.Taxon.TAX_RNK_ID ASC ";
358

    
359

    
360
		String countQuery =
361
			selectCount + fromClause;
362

    
363
		String selectQuery =
364
			selectColumns + fromClause + orderClause;
365

    
366
		if(logger.isInfoEnabled()) { logger.info("Start making misapplied name relationships..."); }
367

    
368
		try {
369

    
370
			ResultSet rs = source.getResultSet(countQuery);
371
			rs.next();
372
			int count = rs.getInt(1);
373

    
374
			rs = source.getResultSet(selectQuery);
375

    
376
	        if (logger.isInfoEnabled()) {
377
				logger.info("Number of rows: " + count);
378
				logger.info("Count Query: " + countQuery);
379
				logger.info("Select Query: " + selectQuery);
380
			}
381

    
382
			while (rs.next()) {
383

    
384
				if ((i++ % limit) == 0) {
385

    
386
					txStatus = startTransaction();
387
					childParentMap = new HashMap<UUID, UUID>(limit);
388

    
389
					if(logger.isInfoEnabled()) {
390
						logger.info("Misapplied names retrieved: " + (i-1) );
391
					}
392
				}
393

    
394
				String childUuidStr = rs.getString("MisappliedUuid");
395
				String parentUuidStr = rs.getString("AcceptedUuid");
396
				UUID childUuid = UUID.fromString(childUuidStr);
397
				UUID parentUuid = UUID.fromString(parentUuidStr);
398

    
399
				if (!childParentMap.containsKey(childUuid)) {
400
						childParentMap.put(childUuid, parentUuid);
401
				} else {
402
					if(logger.isDebugEnabled()) {
403
						logger.debug("Duplicated child UUID (" + childUuid + ")");
404
					}
405
				}
406

    
407
				if (((i % limit) == 0 && i != 1 ) || i == count) {
408

    
409
					createAndCommitMisappliedNameRelationships(state, txStatus,
410
							childParentMap);
411
					childParentMap = null;
412

    
413

    
414
					if(logger.isInfoEnabled()) {
415
						logger.info("i = " + i + " - Transaction committed");
416
					}
417
				}
418
			}
419
			if (childParentMap != null){
420
				logger.info("processMisappliedNames... last commit");
421
				createAndCommitMisappliedNameRelationships(state, txStatus,
422
						childParentMap);
423
				childParentMap = null;
424
			}
425

    
426
		} catch (SQLException e) {
427
			logger.error("SQLException:" +  e);
428
			state.setUnsuccessfull();
429
		}
430
		return;
431
	}
432

    
433
	private void createAndCommitMisappliedNameRelationships(
434
			FaunaEuropaeaImportState state, TransactionStatus txStatus,
435
			Map<UUID, UUID> childParentMap) {
436
		createMisappliedNameRelationships(state, childParentMap);
437
		commitTransaction(txStatus);
438
	}
439

    
440
	/** Retrieve synonyms from FauEuDB DB */
441
	private void processHeterotypicSynonyms(FaunaEuropaeaImportState state, String fromClause) {
442

    
443
		FaunaEuropaeaImportConfigurator fauEuConfig = state.getConfig();
444
		Source source = fauEuConfig.getSource();
445

    
446
		String selectCount =
447
			" SELECT count(*) ";
448

    
449
		String selectColumns = " SELECT Taxon.UUID AS SynonymUuid, Parent.UUID AS AcceptedUuid ";
450

    
451
		String orderClause = " ORDER BY dbo.Taxon.TAX_RNK_ID ASC ";
452

    
453
		String countQuery =
454
			selectCount + fromClause;
455

    
456
		String selectQuery =
457
			selectColumns + fromClause + orderClause;
458
		logger.debug(selectQuery);
459

    
460
		try {
461

    
462
			ResultSet rs = source.getResultSet(countQuery);
463
			rs.next();
464
			int count = rs.getInt(1);
465

    
466
			rs = source.getResultSet(selectQuery);
467

    
468
	        if (logger.isInfoEnabled()) {
469
				logger.info("Number of rows: " + count);
470
				logger.info("Count Query: " + countQuery);
471
				logger.info("Select Query: " + selectQuery);
472
			}
473

    
474
	        storeSynonymRelationships(rs, count, state);
475
	        rs = null;
476
		} catch (SQLException e) {
477
			logger.error("SQLException:" +  e);
478
			state.setUnsuccessfull();
479
		}
480
		return;
481
	}
482

    
483
	private void storeSynonymRelationships(ResultSet rs, int count, FaunaEuropaeaImportState state)
484
	throws SQLException {
485

    
486
		TransactionStatus txStatus = null;
487
		Map<UUID, UUID> synonymAcceptedMap = null;
488
		int i = 0;
489
		int limit = state.getConfig().getLimitSave();
490

    
491
		while (rs.next()) {
492

    
493
			if ((i++ % limit) == 0) {
494

    
495
				txStatus = startTransaction();
496
				synonymAcceptedMap = new HashMap<UUID, UUID>(limit);
497

    
498
				if(logger.isInfoEnabled()) {
499
					logger.info("Synonyms retrieved: " + (i-1));
500
				}
501
			}
502

    
503
			String synonymUuidStr = rs.getString("SynonymUuid");
504
			String acceptedUuidStr = rs.getString("AcceptedUuid");
505
			UUID synonymUuid = UUID.fromString(synonymUuidStr);
506
			UUID acceptedUuid = UUID.fromString(acceptedUuidStr);
507

    
508
			if (!synonymAcceptedMap.containsKey(synonymUuid)) {
509

    
510
				synonymAcceptedMap.put(synonymUuid, acceptedUuid);
511

    
512
			} else {
513
				if(logger.isDebugEnabled()) {
514
					logger.debug("Duplicated synonym UUID (" + synonymUuid + ")");
515
				}
516
			}
517

    
518
			if (((i % limit) == 0 && i != 1 ) || i == count ) {
519

    
520
				synonymAcceptedMap = createAndCommitHeterotypicSynonyms(state,
521
						txStatus, synonymAcceptedMap);
522

    
523
				if(logger.isInfoEnabled()) {
524
					logger.info("i = " + i + " - Transaction committed");
525
				}
526
			}
527
		}
528
		if (synonymAcceptedMap != null){
529
			logger.info("processHeterotypicSynonyms... last commit");
530
			synonymAcceptedMap = createAndCommitHeterotypicSynonyms(state,
531
					txStatus, synonymAcceptedMap);
532
		}
533
		return;
534
	}
535

    
536
	private Map<UUID, UUID> createAndCommitHeterotypicSynonyms(
537
			FaunaEuropaeaImportState state, TransactionStatus txStatus,
538
			Map<UUID, UUID> synonymAcceptedMap) {
539
		createHeterotypicSynonyms(state, synonymAcceptedMap);
540

    
541
		synonymAcceptedMap = null;
542
		commitTransaction(txStatus);
543
		return synonymAcceptedMap;
544
	}
545

    
546
	/* Creates parent-child relationships.
547
	 * Parent-child pairs are retrieved in blocks via findByUUID(Set<UUID>) from CDM DB.
548
	 */
549
	private void createParentChildRelationships(FaunaEuropaeaImportState state, Map<UUID, UUID> childParentMap, Map<UUID, UUID> taxonSpecialistMap, Map<UUID, UUID> taxonGroupCoordinatorMap, TransactionStatus tx) {
550
		//gets the taxon "Hydroscaphidae"(family)
551
		TaxonBase<?> taxon = getTaxonService().find(UUID.fromString(acceptedTaxonUUID));
552
		sourceRef = taxon.getSec();
553
		int limit = state.getConfig().getLimitSave();
554

    
555
		Classification tree = getClassificationFor(state, sourceRef);
556

    
557
		Set<TaxonBase> childSet = new HashSet<>(limit);
558

    
559
		Set<UUID> childKeysSet = childParentMap.keySet();
560
		Set<UUID> parentValuesSet = new HashSet<>(childParentMap.values());
561
		logger.debug("Start reading children and parents");
562
		if (logger.isInfoEnabled()) {
563
			logger.info("Start reading children and parents");
564
		}
565
		List<TaxonBase> children = getTaxonService().find(childKeysSet);
566
		List<TaxonBase> parents = getTaxonService().find(parentValuesSet);
567
		logger.info(children.size() + "children are available");
568
		logger.info(parents.size() + "parents are available");
569
		Map<UUID, TaxonBase> parentsMap = new HashMap<>(parents.size());
570

    
571
		for (TaxonBase<?> taxonBase : parents){
572
			parentsMap.put(taxonBase.getUuid(), taxonBase);
573
		}
574

    
575
		if (logger.isTraceEnabled()) {
576
			logger.debug("End reading children and parents");
577
			for (UUID uuid : childKeysSet) {
578
				logger.trace("child uuid query: " + uuid);
579
			}
580
			for (UUID uuid : parentValuesSet) {
581
				logger.trace("parent uuid query: " + uuid);
582
			}
583
			for (TaxonBase<?> tb : children) {
584
				logger.trace("child uuid result: " + tb.getUuid());
585
			}
586
			for (TaxonBase<?> tb : parents) {
587
				logger.trace("parent uuid result: " + tb.getUuid());
588
			}
589
		}
590

    
591
		UUID mappedParentUuid = null;
592
		UUID childUuid = null;
593
		TaxonNode childNode;
594
		TeamOrPersonBase taxonomicSpecialist = null;
595
		TeamOrPersonBase groupCoordinator = null;
596
		UUID agentUuid = null;
597
		for (TaxonBase child : children) {
598

    
599
			try {
600
				Taxon childTaxon = child.deproxy(child, Taxon.class);
601
				childUuid = childTaxon.getUuid();
602
				mappedParentUuid = childParentMap.get(childUuid);
603
				TaxonBase parent = null;
604

    
605
				TaxonBase potentialParent = parentsMap.get(mappedParentUuid);
606
//					for (TaxonBase potentialParent : parents ) {
607
//						parentUuid = potentialParent.getUuid();
608
//						if(parentUuid.equals(mappedParentUuid)) {
609
						parent = potentialParent;
610
						if (logger.isInfoEnabled()) {
611
							logger.info("Parent (" + mappedParentUuid + ") found for child (" + childUuid + ")");
612
						}
613
//							break;
614
//						}
615
//					}
616

    
617
				Taxon parentTaxon = parent.deproxy(parent, Taxon.class);
618

    
619
				if (childTaxon != null && parentTaxon != null) {
620

    
621
					childNode = tree.addParentChild(parentTaxon, childTaxon, sourceRef, null);
622
					agentUuid = taxonSpecialistMap.get(childTaxon.getUuid());
623

    
624
					taxonomicSpecialist = agentMap.get(agentUuid);
625
					if (taxonomicSpecialist == null){
626
					    taxonomicSpecialist = (TeamOrPersonBase) getAgentService().find(agentUuid);
627
					    if (taxonomicSpecialist != null){
628
					        agentMap.put(agentUuid, taxonomicSpecialist);
629
					        logger.info("get new person: " + agentUuid + " name: " + taxonomicSpecialist.getTitleCache());
630
					    }
631
					}
632
					if (taxonomicSpecialist != null){
633
    				    childNode.addAgentRelation(FaunaEuropaeaTransformer.getTaxonomicSpecialistType(getTermService()),  taxonomicSpecialist);
634
    				} else {
635
    				    logger.info("Taxonomic Specialist for " + childTaxon.getTitleCache() + " - " + childTaxon.getUuid() + " does not exist");
636
    				}
637

    
638

    
639
					agentUuid = taxonGroupCoordinatorMap.get(childTaxon.getUuid());
640
					groupCoordinator = agentMap.get(agentUuid);
641

    
642
					if (groupCoordinator == null){
643
					    groupCoordinator = (TeamOrPersonBase) getAgentService().find(agentUuid);
644
					    if (groupCoordinator != null){
645
					        agentMap.put(agentUuid, groupCoordinator);
646
					        logger.info("get new person: " + agentUuid + " name: " + groupCoordinator.getTitleCache());
647
					    }
648
					}
649
					if (groupCoordinator != null){
650
                        childNode.addAgentRelation(FaunaEuropaeaTransformer.getGroupCoordinatorType(getTermService()),  groupCoordinator);
651
                    } else {
652
                        logger.debug("Group Coordinator for " + childTaxon.getTitleCache() + " - " + childTaxon.getUuid() + " does not exist");
653
                    }
654

    
655
					//TODO: add the specialist and the group coordinator to the node!!
656
					if (logger.isInfoEnabled()) {
657
						logger.info("Parent-child (" + mappedParentUuid + "-" + childUuid +
658
						") relationship created");
659
					}
660
					if (childTaxon != null && !childSet.contains(childTaxon)) {
661

    
662
						childSet.add(childTaxon);
663

    
664
						if (logger.isDebugEnabled()) {
665
							logger.info("Child taxon (" + childUuid + ") added to Set");
666
						}
667

    
668
					} else {
669
						if (logger.isDebugEnabled()) {
670
							logger.info("Duplicated child taxon (" + childUuid + ")");
671
						}
672
					}
673
				} else {
674
					if (logger.isDebugEnabled()) {
675
						logger.info("Parent(" + mappedParentUuid + ") or child (" + childUuid + " is null");
676
					}
677
				}
678

    
679
			} catch (Exception e) {
680
				logger.error("Error creating taxonomically included relationship parent-child (" +
681
					mappedParentUuid + " <----> " + childUuid + ")", e);
682
			}
683

    
684
		}
685
		if (logger.isInfoEnabled()) {
686
			logger.info("Start saving childSet");
687
		}
688

    
689
		getTaxonService().saveOrUpdate(childSet);
690

    
691
		if (logger.isInfoEnabled()) {
692
			logger.info("End saving childSet");
693
		}
694

    
695
		parentValuesSet = null;
696
		childSet = null;
697
		children = null;
698
		parents = null;
699
		tree = null;
700
		this.agentMap = new HashMap<UUID, TeamOrPersonBase>();
701
		return;
702
	}
703

    
704
	/* Creates misapplied name relationships.
705
	 * Misapplied name-accepted taxon pairs are retrieved in blocks via findByUUID(Set<UUID>) from CDM DB.
706
	 */
707
	private void createMisappliedNameRelationships(FaunaEuropaeaImportState state, Map<UUID, UUID> fromToMap) {
708

    
709
		//gets the taxon "Hydroscaphidae" (family)
710

    
711
		TaxonBase taxon = getTaxonService().find(UUID.fromString(acceptedTaxonUUID));
712
		sourceRef = taxon.getSec();
713
		int limit = state.getConfig().getLimitSave();
714

    
715
			Set<TaxonBase> misappliedNameSet = new HashSet<TaxonBase>(limit);
716

    
717
			Set<UUID> misappliedNamesSet = fromToMap.keySet();
718
			Set<UUID> acceptedTaxaSet = new HashSet<UUID>(fromToMap.values());
719

    
720
			if (logger.isTraceEnabled()) {
721
				logger.trace("Start reading misapplied names and accepted taxa");
722
			}
723
			List<TaxonBase> misappliedNames = getTaxonService().find(misappliedNamesSet);
724
			List<TaxonBase> acceptedTaxa = getTaxonService().find(acceptedTaxaSet);
725
			Map<UUID, TaxonBase> acceptedTaxaMap = new HashMap<UUID, TaxonBase>(acceptedTaxa.size());
726
			for (TaxonBase taxonBase : acceptedTaxa){
727
				acceptedTaxaMap.put(taxonBase.getUuid(), taxonBase);
728
			}
729

    
730
			if (logger.isTraceEnabled()) {
731
				logger.info("End reading misapplied names and accepted taxa");
732
				for (UUID uuid : misappliedNamesSet) {
733
					logger.trace("misapplied name uuid query: " + uuid);
734
				}
735
				for (UUID uuid : acceptedTaxaSet) {
736
					logger.trace("accepted taxon uuid query: " + uuid);
737
				}
738
				for (TaxonBase tb : misappliedNames) {
739
					logger.trace("misapplied name uuid result: " + tb.getUuid());
740
				}
741
				for (TaxonBase tb : acceptedTaxa) {
742
					logger.trace("accepted taxon uuid result: " + tb.getUuid());
743
				}
744
			}
745

    
746
			UUID mappedAcceptedTaxonUuid = null;
747
			UUID misappliedNameUuid = null;
748
			Taxon misappliedNameTaxon = null;
749
			TaxonBase acceptedTaxonBase = null;
750
			Taxon acceptedTaxon = null;
751

    
752
			for (TaxonBase misappliedName : misappliedNames) {
753

    
754
				try {
755

    
756
					misappliedNameTaxon = misappliedName.deproxy(misappliedName, Taxon.class);
757
					misappliedName.setAppendedPhrase(AUCT_STRING);
758

    
759
					misappliedNameUuid = misappliedNameTaxon.getUuid();
760
					mappedAcceptedTaxonUuid = fromToMap.get(misappliedNameUuid);
761
					acceptedTaxonBase = null;
762

    
763
					acceptedTaxonBase = acceptedTaxaMap.get(mappedAcceptedTaxonUuid);
764
							if (logger.isDebugEnabled()) {
765
								logger.debug("Parent (" + mappedAcceptedTaxonUuid + ") found for child (" + misappliedNameUuid + ")");
766
							}
767

    
768
							acceptedTaxon = acceptedTaxonBase.deproxy(acceptedTaxonBase, Taxon.class);
769

    
770
					if (misappliedNameTaxon != null && acceptedTaxon != null) {
771

    
772
						acceptedTaxon.addMisappliedName(misappliedNameTaxon, sourceRef, null);
773

    
774
						if (logger.isDebugEnabled()) {
775
							logger.debug("Accepted taxon / misapplied name (" + mappedAcceptedTaxonUuid + "-" + misappliedNameUuid +
776
							") relationship created");
777
						}
778
						if (!misappliedNameSet.contains(misappliedNameTaxon)) {
779

    
780
							misappliedNameSet.add(misappliedNameTaxon);
781

    
782
							if (logger.isTraceEnabled()) {
783
								logger.trace("Misapplied name taxon (" + misappliedNameUuid + ") added to Set");
784
							}
785

    
786
						} else {
787
							if (logger.isDebugEnabled()) {
788
								logger.debug("Duplicated misapplied name taxon (" + misappliedNameUuid + ")");
789
							}
790
						}
791
					} else {
792
						if (logger.isDebugEnabled()) {
793
							logger.debug("Accepted taxon (" + mappedAcceptedTaxonUuid + ") or misapplied name (" + misappliedNameUuid + " is null");
794
						}
795
					}
796

    
797
					if (misappliedNameTaxon != null && !misappliedNameSet.contains(misappliedNameTaxon)) {
798
						misappliedNameSet.add(misappliedNameTaxon);
799
						if (logger.isTraceEnabled()) {
800
							logger.trace("Misapplied name taxon (" + misappliedNameUuid + ") added to Set");
801
						}
802
					} else {
803
						if (logger.isDebugEnabled()) {
804
							logger.debug("Duplicated misapplied name taxon (" + misappliedNameUuid + ")");
805
						}
806
					}
807

    
808
				} catch (Exception e) {
809
					logger.error("Error creating misapplied name relationship accepted taxon-misapplied name (" +
810
						mappedAcceptedTaxonUuid + "-" + misappliedNameUuid + ")", e);
811
				}
812

    
813
			}
814
			if (logger.isTraceEnabled()) {
815
				logger.trace("Start saving misappliedNameSet");
816
			}
817
			getTaxonService().save(misappliedNameSet);
818
			if (logger.isTraceEnabled()) {
819
				logger.trace("End saving misappliedNameSet");
820
			}
821

    
822
			acceptedTaxaSet = null;
823
			misappliedNameSet = null;
824
			misappliedNames = null;
825
			acceptedTaxa = null;
826

    
827
		return;
828
	}
829

    
830

    
831
	/* Creates heterotypic synonyms.
832
	 * Synonym-accepted taxon pairs are retrieved in blocks via findByUUID(Set<UUID>) from CDM DB.
833
	 */
834
	private void createHeterotypicSynonyms(FaunaEuropaeaImportState state, Map<UUID, UUID> fromToMap) {
835

    
836
		int limit = state.getConfig().getLimitSave();
837

    
838
		Set<TaxonBase> synonymSet = new HashSet<TaxonBase>(limit);
839

    
840
		Set<UUID> synonymUuidSet = fromToMap.keySet();
841
		Set<UUID> acceptedTaxaUuidSet = new HashSet<UUID>(fromToMap.values());
842

    
843
		if (logger.isTraceEnabled()) {
844
			logger.trace("Reading synonym names and accepted taxa...");
845
		}
846
		List<TaxonBase> synonyms = getTaxonService().find(synonymUuidSet);
847
		List<TaxonBase> acceptedTaxa = getTaxonService().find(acceptedTaxaUuidSet);
848
		Map<UUID, TaxonBase> acceptedTaxaMap = new HashMap<UUID, TaxonBase>(acceptedTaxa.size());
849
		for (TaxonBase taxonBase : acceptedTaxa){
850
			acceptedTaxaMap.put(taxonBase.getUuid(), taxonBase);
851
		}
852

    
853
		if (logger.isTraceEnabled()) {
854
			logger.trace("End reading synonyms names and accepted taxa");
855
			for (UUID uuid : synonymUuidSet) {
856
				logger.trace("synonym uuid query: " + uuid);
857
			}
858
			for (UUID uuid : acceptedTaxaUuidSet) {
859
				logger.trace("accepted taxon uuid query: " + uuid);
860
			}
861
			for (TaxonBase tb : synonyms) {
862
				logger.trace("synonym uuid result: " + tb.getUuid());
863
			}
864
			for (TaxonBase tb : acceptedTaxa) {
865
				logger.trace("accepted taxon uuid result: " + tb.getUuid());
866
			}
867
		}
868

    
869
		UUID mappedAcceptedTaxonUuid = null;
870
		UUID synonymUuid = null;
871
		Synonym synonym = null;
872
		TaxonBase acceptedTaxonBase = null;
873
		Taxon acceptedTaxon = null;
874

    
875
		for (TaxonBase synonymTaxonBase : synonyms) {
876

    
877
			try {
878
				//check for misapplied names with nec (in Fauna Europaea they have a synonym relationship to the accepted taxon)
879
				if (synonymTaxonBase instanceof Taxon ){
880
					if (((Taxon)synonymTaxonBase).isMisapplication()){
881
						if (logger.isDebugEnabled()) {
882
							logger.debug("misapplied name with exclusion" +  synonymTaxonBase.getTitleCache());
883
						}
884
					}
885
					continue;
886
				}
887
				synonym = HibernateProxyHelper.deproxy(synonymTaxonBase, Synonym.class);
888
				synonymUuid = synonym.getUuid();
889
				mappedAcceptedTaxonUuid = fromToMap.get(synonymUuid);
890
				acceptedTaxonBase = null;
891

    
892
				acceptedTaxonBase = acceptedTaxaMap.get(mappedAcceptedTaxonUuid);
893
				if (logger.isDebugEnabled()) {
894
					logger.debug("Parent (" + mappedAcceptedTaxonUuid + ") found for child (" + synonymUuid + ")");
895
				}
896
				acceptedTaxon = CdmBase.deproxy(acceptedTaxonBase, Taxon.class);
897

    
898
				if (synonym != null && acceptedTaxon != null) {
899

    
900
					//TODO: in case original genus exists must add synonym to original genus instead of to accepted taxon
901
					acceptedTaxon.addSynonym(synonym, SynonymType.HETEROTYPIC_SYNONYM_OF);
902

    
903
					if (logger.isDebugEnabled()) {
904
						logger.debug("Accepted taxon - synonym (" + mappedAcceptedTaxonUuid + " - " + synonymUuid +
905
						") relationship created");
906
					}
907
					if (synonym != null && !synonymSet.contains(synonym)) {
908

    
909
						synonymSet.add(synonym);
910

    
911
						if (logger.isTraceEnabled()) {
912
							logger.trace("Synonym (" + synonymUuid + ") added to Set");
913
						}
914

    
915
					} else {
916
						if (logger.isDebugEnabled()) {
917
							logger.debug("Duplicated synonym (" + synonymUuid + ")");
918
						}
919
					}
920
				} else {
921
					if (logger.isDebugEnabled()) {
922
						logger.debug("Accepted taxon (" + mappedAcceptedTaxonUuid + ") or misapplied name (" + synonymUuid + " is null");
923
					}
924
				}
925
			} catch (Exception e) {
926
				logger.error("Error attaching synonym to accepted taxon (" +
927
						mappedAcceptedTaxonUuid + "-" + synonymUuid + ": "+ synonymTaxonBase.getTitleCache() +")", e);
928
			}
929
		}
930
		if (logger.isTraceEnabled()) {
931
			logger.trace("Start saving synonymSet");
932
		}
933
		getTaxonService().save(synonymSet);
934
		if (logger.isTraceEnabled()) {
935
			logger.trace("End saving synonymSet");
936
		}
937

    
938
		acceptedTaxaUuidSet = null;
939
		synonymSet = null;
940
		synonyms = null;
941
		acceptedTaxa = null;
942

    
943
		return;
944
	}
945

    
946
	private void processAssociatedSpecialists(FaunaEuropaeaImportState state){
947
		int limit = state.getConfig().getLimitSave();
948

    
949
		TransactionStatus txStatus = null;
950
		Set<UUID> taxonUuids = new HashSet<UUID>();
951
		Map<UUID, Set<UUID>> agentUUIDTaxonMap = new HashMap<UUID, Set<UUID>>();
952
		FaunaEuropaeaImportConfigurator fauEuConfig = state.getConfig();
953
		Source source = fauEuConfig.getSource();
954
		int i = 0;
955
		UUID agentUuid;
956
		Set<UUID> agentUuids = new HashSet<UUID>();
957
		Set<UUID> taxaOfAgent = new HashSet<UUID>();
958
		String selectCount =
959
			" SELECT count(*) ";
960

    
961

    
962

    
963
		String selectColumns = "SELECT  u.USR_GROUPNAME as groupName, u.USR_GROUPNOTE groupNote, u.USR_USR_ID as user_user_id, "
964
				+ "		u.usr_id user_id, user2.USR_ID as user2_id, taxon.UUID as tax_uuid ";
965

    
966
		String fromClause = " FROM USERROLE as userrole left join USERS u on userrole.URL_USR_ID = u.USR_ID join USERS user2 on user2.USR_ID = u.USR_USR_ID "
967
						+ " join TAXON taxon on (taxon.TAX_USR_IDSP= user2.USR_ID or taxon.TAX_USR_IDGC= user2.USR_ID) "
968
						+ " where USERROLE.URL_ROL_ID = 7 ";
969
		String orderClause = " ORDER BY taxon.TAX_ID";
970

    
971
		String countQuery =
972
			selectCount + fromClause;
973

    
974
		String selectQuery =
975
			selectColumns + fromClause + orderClause;
976

    
977
		if(logger.isInfoEnabled()) { logger.info("Start making associated specialists..."); }
978

    
979
		try {
980

    
981
			ResultSet rs = source.getResultSet(countQuery);
982
			rs.next();
983
			int count = rs.getInt(1);
984

    
985
			rs = source.getResultSet(selectQuery);
986

    
987
	        if (logger.isInfoEnabled()) {
988
				logger.info("Number of rows: " + count);
989
				logger.info("Count Query: " + countQuery);
990
				logger.info("Select Query: " + selectQuery);
991
			}
992
	        String oldTaxonUuidString = null;
993
	        while (rs.next()) {
994

    
995
                if ((i++ % limit) == 0) {
996
                    txStatus = startTransaction();
997
                    if(logger.isInfoEnabled()) {
998
                        logger.info("Associated specialists: " + (i-1));
999
                    }
1000
                }
1001
                String taxonUUID = rs.getString("tax_uuid");
1002

    
1003
				int userId = rs.getInt("user_id");
1004
				int user2Id = rs.getInt("user2_id");
1005
				String groupName = rs.getString("groupName");
1006

    
1007
				taxonUuids.add(UUID.fromString(taxonUUID));
1008
				agentUuid = state.getAgentMap().get(userId);
1009
				agentUuids.add(agentUuid);
1010
				if (agentUUIDTaxonMap.containsKey(agentUuid)){
1011
				    taxaOfAgent = agentUUIDTaxonMap.get(agentUuid);
1012
				} else{
1013
				    taxaOfAgent = new HashSet<UUID>();
1014
				}
1015
				taxaOfAgent.add(UUID.fromString(taxonUUID));
1016
                agentUUIDTaxonMap.put(agentUuid, taxaOfAgent);
1017

    
1018

    
1019
				 if (((i % limit) == 0 && i != 1 ) || i == count ) {
1020

    
1021
				        createTaxonNodeAgentRealtionships(state, taxonUuids, agentUUIDTaxonMap, agentUuids);
1022
		                commitTransaction(txStatus);
1023
		                agentUUIDTaxonMap.clear();
1024
		                taxonUuids= new HashSet<UUID>();
1025
		                agentUuids=new HashSet<UUID>();
1026
		                if(logger.isInfoEnabled()) {
1027
		                    logger.info("i = " + i + " - Transaction committed");
1028
		                }
1029
		            }
1030

    
1031
	        }
1032
		}catch(SQLException e){
1033
			logger.info("Problems during creating associated specialists.", e);
1034
		}
1035
	}
1036

    
1037
	private void createTaxonNodeAgentRealtionships(FaunaEuropaeaImportState state, Set<UUID> taxonUuids, Map<UUID, Set<UUID>> agentUUIDTaxonMap, Set<UUID> agentUUuids){
1038
	    Taxon taxon = null;
1039
	    TaxonBase taxonBase;
1040
        Synonym syn;
1041
        Set<TaxonNode> nodes = new HashSet<TaxonNode>();
1042
        List<TaxonBase> taxonBases = getTaxonService().find(taxonUuids);
1043
        List<AgentBase> agentBases = getAgentService().find(agentUUuids);
1044
        TeamOrPersonBase associatedSpecialist = null;
1045
        TaxonNodeAgentRelation agentRel = null;
1046

    
1047
        TaxonNode taxonNode;
1048
//        Map<UUID, TeamOrPersonBase> agentsMap = new HashMap<UUID,TeamOrPersonBase>();
1049
//        for (AgentBase agent: agentBases){
1050
//            agentsMap.put(agent.getUuid(), (TeamOrPersonBase) agent);
1051
//        }
1052
        Map<UUID, TaxonBase> taxonMap = new HashMap<UUID,TaxonBase>();
1053
        for (TaxonBase tax: taxonBases){
1054
            taxonMap.put(tax.getUuid(), tax);
1055
        }
1056

    
1057
        for (AgentBase agent: agentBases){
1058
            Set<UUID> taxaUuids = agentUUIDTaxonMap.get(agent.getUuid());
1059
            for (UUID taxonUuid: taxaUuids){
1060
                taxonBase = taxonMap.get(taxonUuid);
1061
                    if (taxonBase instanceof Synonym){
1062
                        logger.debug("synonym has relationship to associated specialist. " + taxonBase.getTitleCache());
1063
                        syn = HibernateProxyHelper.deproxy(taxonBase, Synonym.class);
1064
                        if (syn.getAcceptedTaxon() != null){
1065
                            taxon = syn.getAcceptedTaxon();
1066
                            syn = null;
1067
                        }
1068
                    } else{
1069
                        taxon = HibernateProxyHelper.deproxy(taxonBase, Taxon.class);
1070
                    }
1071

    
1072
                    if (taxon != null && !taxon.getTaxonNodes().isEmpty()){
1073
                        taxonNode = taxon.getTaxonNodes().iterator().next();
1074
                    } else {
1075
                        taxonNode = null;
1076
                        logger.debug("There is an associated specialist for a taxon which has no taxonnode.");
1077
                    }
1078

    
1079

    
1080
                    associatedSpecialist = (TeamOrPersonBase)agent;
1081
                    if (associatedSpecialist != null && taxonNode != null){
1082
                        agentRel =taxonNode.addAgentRelation(FaunaEuropaeaTransformer.getAssociateSpecialistType(getTermService()), associatedSpecialist);
1083
                        /*if (!StringUtils.isBlank(groupName)) {
1084
                            agentRel.addAnnotation(Annotation.NewInstance(groupName, Language.DEFAULT()));
1085
                        }*/
1086
                    }
1087

    
1088

    
1089

    
1090
                nodes.add(taxonNode);
1091
              }
1092
            agent = null;
1093
            associatedSpecialist = null;
1094
            taxonNode = null;
1095
        }
1096
        if (!nodes.isEmpty()){
1097
            while (nodes.contains(null)){
1098
                nodes.remove(null);
1099
            }
1100
            getTaxonNodeService().saveOrUpdate(nodes);
1101
            nodes = null;
1102

    
1103
        }
1104
        taxonMap= null;
1105
        agentMap= null;
1106
        taxonBases = null;
1107
        agentBases = null;
1108
       }
1109

    
1110

    
1111

    
1112
	private void processInferredSynonyms(FaunaEuropaeaImportState state){
1113

    
1114
	        int count;
1115
	        int pastCount;
1116
	        boolean success = true;
1117
	        // Get the limit for objects to save within a single transaction.
1118
	        if (! state.getConfig().isDoInferredSynonyms()){
1119
	            logger.info ("Ignore Creating Inferred Synonyms...");
1120
	            return;
1121
	        }
1122

    
1123
	        int limit = state.getConfig().getLimitSave();
1124
	        // Create inferred synonyms for accepted taxa
1125
	        logger.info("Creating Inferred Synonyms...");
1126

    
1127

    
1128
	        count = 0;
1129
	        pastCount = 0;
1130
	        int pageSize = limit/10;
1131
	        int pageNumber = 1;
1132
	        String inferredSynonymPluralString = "Inferred Synonyms";
1133

    
1134
	        // Start transaction
1135
	        TransactionStatus txStatus = startTransaction(true);
1136
	        logger.info("Started new transaction. Fetching some " + parentPluralString + " first (max: " + limit + ") ...");
1137
	        List<Taxon> taxonList = null;
1138
	        Set<TaxonBase> synonymList = new HashSet<>();
1139

    
1140

    
1141
	        while ((taxonList  = getTaxonService().listTaxaByName(Taxon.class, "*", "*", "*", "*", "*", Rank.SPECIES(), pageSize, pageNumber, null)).size() > 0) {
1142
	            HashMap<Integer, TaxonName> inferredSynonymsDataToBeSaved = new HashMap<>();
1143

    
1144
	            logger.info("Fetched " + taxonList.size() + " " + parentPluralString + ". Importing...");
1145
	            synonymList = createInferredSynonymsForTaxonList(state,  taxonList);
1146
	            getTaxonService().save(synonymList);
1147

    
1148
	          //  getTaxonService().saveOrUpdate(taxonList);
1149
	            // Commit transaction
1150
	            commitTransaction(txStatus);
1151
	            logger.debug("Committed transaction.");
1152
	            logger.info("Imported " + (taxonList.size()) + " " + inferredSynonymPluralString + ". Total: " + count);
1153
	            //pastCount = count;
1154

    
1155

    
1156

    
1157
	            // Start transaction
1158
	            txStatus = startTransaction(true);
1159
	            logger.info("Started new transaction. Fetching some " + parentPluralString + " first (max: " + limit + ") ...");
1160

    
1161
	            // Increment pageNumber
1162
	            pageNumber++;
1163
	        }
1164
	        taxonList = null;
1165
	        while ((taxonList  = getTaxonService().listTaxaByName(Taxon.class, "*", "*", "*", "*", "*", Rank.SUBSPECIES(), pageSize, pageNumber, null)).size() > 0) {
1166
	            HashMap<Integer, TaxonName> inferredSynonymsDataToBeSaved = new HashMap<>();
1167

    
1168
	            logger.info("Fetched " + taxonList.size() + " " + parentPluralString  + ". Exporting...");
1169
	            synonymList = createInferredSynonymsForTaxonList(state, taxonList);
1170

    
1171
	            getTaxonService().save(synonymList);
1172
	            // Commit transaction
1173
	            commitTransaction(txStatus);
1174
	            logger.debug("Committed transaction.");
1175
	            logger.info("Exported " + taxonList.size()+ " " + inferredSynonymPluralString + ". Total: " + count);
1176
	            //pastCount = count;
1177

    
1178

    
1179

    
1180
	            // Start transaction
1181
	            txStatus = startTransaction(true);
1182
	            logger.info("Started new transaction. Fetching some " + parentPluralString + " first (max: " + limit + ") ...");
1183

    
1184
	            // Increment pageNumber
1185
	            pageNumber++;
1186
	            inferredSynonymsDataToBeSaved = null;
1187
	        }
1188
	        if (taxonList.size() == 0) {
1189
	            logger.info("No " + parentPluralString + " left to fetch.");
1190
	        }
1191

    
1192
	        taxonList = null;
1193

    
1194
	        // Commit transaction
1195
	        commitTransaction(txStatus);
1196
	        System.gc();
1197

    
1198
	        //ProfilerController.memorySnapshot();
1199
	        logger.debug("Committed transaction.");
1200

    
1201

    
1202
	}
1203

    
1204
	/**
1205
    * @param state
1206
    * @param mapping
1207
    * @param synRelMapping
1208
    * @param currentTaxonId
1209
    * @param taxonList
1210
    * @param inferredSynonymsDataToBeSaved
1211
    * @return
1212
    */
1213
   private Set<TaxonBase> createInferredSynonymsForTaxonList(FaunaEuropaeaImportState state,
1214
            List<Taxon> taxonList) {
1215

    
1216
       Taxon acceptedTaxon;
1217
       Classification classification = null;
1218
       Set<TaxonBase> inferredSynonyms = new HashSet<>();
1219
       List<Synonym> inferredSynonymsLocal= new ArrayList<>();
1220
       boolean localSuccess = true;
1221

    
1222
       HashMap<Integer, TaxonName> inferredSynonymsDataToBeSaved = new HashMap<>();
1223

    
1224
       for (TaxonBase<?> taxonBase : taxonList) {
1225

    
1226
           if (taxonBase.isInstanceOf(Taxon.class)) { // this should always be the case since we should have fetched accepted taxon only, but you never know...
1227
               acceptedTaxon = CdmBase.deproxy(taxonBase, Taxon.class);
1228
               TaxonName taxonName = acceptedTaxon.getName();
1229

    
1230
               if (taxonName.isZoological()) {
1231
                   Set<TaxonNode> taxonNodes = acceptedTaxon.getTaxonNodes();
1232
                   TaxonNode singleNode = null;
1233

    
1234
                   if (taxonNodes.size() > 0) {
1235
                       // Determine the classification of the current TaxonNode
1236

    
1237
                       singleNode = taxonNodes.iterator().next();
1238
                       if (singleNode != null) {
1239
                           classification = singleNode.getClassification();
1240
                       } else {
1241
                           logger.error("A TaxonNode belonging to this accepted Taxon is NULL: " + acceptedTaxon.getUuid() + " (" + acceptedTaxon.getTitleCache() +")");
1242
                       }
1243
                   } else {
1244
                       // Classification could not be determined directly from this TaxonNode
1245
                       // The stored classification from another TaxonNode is used. It's a simple, but not a failsafe fallback solution.
1246
                       if (taxonNodes.size() == 0) {
1247
                           //logger.error("Classification could not be determined directly from this Taxon: " + acceptedTaxon.getUuid() + " is misapplication? "+acceptedTaxon.isMisapplication()+ "). The classification of the last taxon is used");
1248

    
1249
                       }
1250
                   }
1251

    
1252
                   if (classification != null) {
1253
                       try{
1254
                           TaxonName name = acceptedTaxon.getName();
1255

    
1256
                            //if (name.isSpecies() || name.isInfraSpecific()){
1257
                               inferredSynonymsLocal = getTaxonService().createAllInferredSynonyms(acceptedTaxon, classification, true);
1258
                              // logger.info("number of inferred synonyms: " + inferredSynonyms.size());
1259
                           //}
1260
//                             inferredSynonyms = getTaxonService().createInferredSynonyms(classification, acceptedTaxon, SynonymType.INFERRED_GENUS_OF());
1261
                           if (inferredSynonymsLocal != null) {
1262
                               for (TaxonBase synonym : inferredSynonymsLocal) {
1263
//                                 TaxonNameBase<?,?> synonymName = synonym.getName();
1264
                                   MarkerType markerType =getUuidMarkerType(PesiTransformer.uuidMarkerGuidIsMissing, state);
1265

    
1266
                                   synonym.addMarker(Marker.NewInstance(markerType, true));
1267

    
1268

    
1269
                                   //get SynonymRelationship and export
1270
                                   if (((Synonym)synonym).getAcceptedTaxon() == null ){
1271
                                       IdentifiableSource source = ((Synonym)synonym).getSources().iterator().next();
1272
                                       if (source.getIdNamespace().contains("Potential combination")){
1273
                                           acceptedTaxon.addSynonym((Synonym)synonym, SynonymType.POTENTIAL_COMBINATION_OF);
1274
                                           logger.error(synonym.getTitleCache() + " is not attached to " + acceptedTaxon.getTitleCache() + " type is set to potential combination");
1275
                                       } else if (source.getIdNamespace().contains("Inferred Genus")){
1276
                                           acceptedTaxon.addSynonym((Synonym)synonym, SynonymType.INFERRED_GENUS_OF);
1277
                                           logger.error(synonym.getTitleCache() + " is not attached to " + acceptedTaxon.getTitleCache() + " type is set to inferred genus");
1278
                                       } else if (source.getIdNamespace().contains("Inferred Epithet")){
1279
                                           acceptedTaxon.addSynonym((Synonym)synonym, SynonymType.INFERRED_EPITHET_OF);
1280
                                           logger.error(synonym.getTitleCache() + " is not attached to " + acceptedTaxon.getTitleCache() + " type is set to inferred epithet");
1281
                                       } else{
1282
                                           acceptedTaxon.addSynonym((Synonym)synonym, SynonymType.INFERRED_SYNONYM_OF);
1283
                                           logger.error(synonym.getTitleCache() + " is not attached to " + acceptedTaxon.getTitleCache() + " type is set to inferred synonym");
1284
                                       }
1285

    
1286
                                   } else {
1287
                                       if (!localSuccess) {
1288
                                           logger.error("Synonym relationship export failed " + synonym.getTitleCache() + " accepted taxon: " + acceptedTaxon.getUuid() + " (" + acceptedTaxon.getTitleCache()+")");
1289
                                       } else {
1290
                                          // logger.info("Synonym relationship successfully exported: " + synonym.getTitleCache() + "  " +acceptedTaxon.getUuid() + " (" + acceptedTaxon.getTitleCache()+")");
1291
                                       }
1292
                                   }
1293

    
1294
                                   inferredSynonymsDataToBeSaved.put(synonym.getId(), synonym.getName());
1295
                               }
1296

    
1297
                               inferredSynonyms.addAll(inferredSynonymsLocal);
1298
                              //logger.info("inferredSet: " + inferredSet.size());
1299
                               //getTaxonService().save(inferredSynonyms);
1300
                               //commitTransaction(txStatus);
1301

    
1302
                               inferredSynonymsLocal = null;
1303

    
1304
                           }
1305

    
1306

    
1307
                       }catch(Exception e){
1308
                           logger.error(e.getMessage());
1309
                           e.printStackTrace();
1310
                       }
1311
                   } else {
1312
                       logger.error("Classification is NULL. Inferred Synonyms could not be created for this Taxon: " + acceptedTaxon.getUuid() + " (" + acceptedTaxon.getTitleCache() + ")");
1313
                   }
1314
               } else {
1315
//                         logger.error("TaxonName is not a ZoologicalName: " + taxonName.getUuid() + " (" + taxonName.getTitleCache() + ")");
1316
               }
1317
           } else {
1318
               logger.error("This TaxonBase is not a Taxon even though it should be: " + taxonBase.getUuid() + " (" + taxonBase.getTitleCache() + ")");
1319
           }
1320
       }
1321
       //getTaxonService().saveOrUpdate(taxonList);
1322
       taxonList = null;
1323
       return inferredSynonyms;
1324
   }
1325

    
1326

    
1327
}
(13-13/20)