Project

General

Profile

« Previous | Next » 

Revision 6243a668

Added by Andreas Müller over 11 years ago

adding collections to AlgaTerra import

View differences:

.gitattributes
98 98
app-import/src/main/java/eu/etaxonomy/cdm/io/PalmaeImageImport.java -text
99 99
app-import/src/main/java/eu/etaxonomy/cdm/io/PalmaeProtologueImport.java -text
100 100
app-import/src/main/java/eu/etaxonomy/cdm/io/ProtologueImport.java -text
101
app-import/src/main/java/eu/etaxonomy/cdm/io/algaterra/AlgaTerraCollectionImport.java -text
101 102
app-import/src/main/java/eu/etaxonomy/cdm/io/algaterra/AlgaTerraImportConfigurator.java -text
102 103
app-import/src/main/java/eu/etaxonomy/cdm/io/algaterra/AlgaTerraImportState.java -text
103 104
app-import/src/main/java/eu/etaxonomy/cdm/io/algaterra/AlgaTerraImportTransformer.java -text
app-import/src/main/java/eu/etaxonomy/cdm/io/algaterra/AlgaTerraCollectionImport.java
1
/**
2
* Copyright (C) 2007 EDIT
3
* European Distributed Institute of Taxonomy 
4
* http://www.e-taxonomy.eu
5
* 
6
* The contents of this file are subject to the Mozilla Public License Version 1.1
7
* See LICENSE.TXT at the top of this package for the full license terms.
8
*/
9

  
10
package eu.etaxonomy.cdm.io.algaterra;
11

  
12
import java.sql.ResultSet;
13
import java.sql.SQLException;
14
import java.util.HashMap;
15
import java.util.HashSet;
16
import java.util.Map;
17
import java.util.Set;
18
import java.util.UUID;
19

  
20
import org.apache.commons.lang.StringUtils;
21
import org.apache.log4j.Logger;
22
import org.springframework.stereotype.Component;
23

  
24
import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
25
import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade.DerivedUnitType;
26
import eu.etaxonomy.cdm.api.facade.DerivedUnitFacadeNotSupportedException;
27
import eu.etaxonomy.cdm.io.algaterra.validation.AlgaTerraCollectionImportValidator;
28
import eu.etaxonomy.cdm.io.algaterra.validation.AlgaTerraSpecimenImportValidator;
29
import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator;
30
import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportState;
31
import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelTaxonImport;
32
import eu.etaxonomy.cdm.io.common.IOValidator;
33
import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
34
import eu.etaxonomy.cdm.model.common.CdmBase;
35
import eu.etaxonomy.cdm.model.common.DefinedTermBase;
36
import eu.etaxonomy.cdm.model.common.Language;
37
import eu.etaxonomy.cdm.model.common.Marker;
38
import eu.etaxonomy.cdm.model.common.MarkerType;
39
import eu.etaxonomy.cdm.model.common.TermVocabulary;
40
import eu.etaxonomy.cdm.model.description.CategoricalData;
41
import eu.etaxonomy.cdm.model.description.DescriptionBase;
42
import eu.etaxonomy.cdm.model.description.Feature;
43
import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
44
import eu.etaxonomy.cdm.model.description.MeasurementUnit;
45
import eu.etaxonomy.cdm.model.description.Modifier;
46
import eu.etaxonomy.cdm.model.description.QuantitativeData;
47
import eu.etaxonomy.cdm.model.description.State;
48
import eu.etaxonomy.cdm.model.description.StatisticalMeasure;
49
import eu.etaxonomy.cdm.model.description.StatisticalMeasurementValue;
50
import eu.etaxonomy.cdm.model.description.TaxonDescription;
51
import eu.etaxonomy.cdm.model.description.TextData;
52
import eu.etaxonomy.cdm.model.name.BotanicalName;
53
import eu.etaxonomy.cdm.model.name.Rank;
54
import eu.etaxonomy.cdm.model.occurrence.Collection;
55
import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
56
import eu.etaxonomy.cdm.model.occurrence.FieldObservation;
57
import eu.etaxonomy.cdm.model.reference.Reference;
58
import eu.etaxonomy.cdm.model.taxon.Taxon;
59
import eu.etaxonomy.cdm.model.taxon.TaxonBase;
60

  
61

  
62
/**
63
 * @author a.mueller
64
 * @created 01.09.2012
65
 */
66
@Component
67
public class AlgaTerraCollectionImport  extends AlgaTerraSpecimenImportBase {
68
	private static final Logger logger = Logger.getLogger(AlgaTerraCollectionImport.class);
69

  
70
	
71
	private static int modCount = 5000;
72
	private static final String pluralString = "collections";
73
	private static final String dbTableName = "Collection";  //??  
74
	
75
	public static final String NAMESPACE_COLLECTION = "Collection"; 
76
	public static final String NAMESPACE_SUBCOLLECTION = "Collection (Subcollection)"; 
77

  
78

  
79
	public AlgaTerraCollectionImport(){
80
		super();
81
	}
82
	
83
	
84
	
85
	/* (non-Javadoc)
86
	 * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getIdQuery()
87
	 */
88
	@Override
89
	protected String getIdQuery(BerlinModelImportState state) {
90
		String result = " SELECT CollectionId " + 
91
				" FROM Collection c "
92
				+ " ORDER BY c.CollectionId ";
93
		return result;
94
	}
95

  
96
	/* (non-Javadoc)
97
	 * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getRecordQuery(eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator)
98
	 */
99
	@Override
100
	protected String getRecordQuery(BerlinModelImportConfigurator config) {
101
			String strQuery =   
102
            " SELECT CollectionId, Name, Town, IHCode, Subcollection, TDWGGazetteerFk, Address, CultCollFlag, " +
103
            		" Created_When, Updated_When, Created_Who, Updated_Who, Notes " +
104
            " FROM Collection c " + 
105
            " WHERE c.CollectionId IN (" + ID_LIST_TOKEN + ") "  
106
            + " ORDER BY c.CollectionId "
107
            ;
108
		return strQuery;
109
	}
110

  
111
	/* (non-Javadoc)
112
	 * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#doPartition(eu.etaxonomy.cdm.io.berlinModel.in.ResultSetPartitioner, eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportState)
113
	 */
114
	public boolean doPartition(ResultSetPartitioner partitioner, BerlinModelImportState bmState) {
115
		boolean success = true;
116
		
117
		AlgaTerraImportState state = (AlgaTerraImportState)bmState;
118
		Set<Collection> collectionsToSave = new HashSet<Collection>();
119
		
120
		ResultSet rs = partitioner.getResultSet();
121

  
122
		try {
123
			
124
			int i = 0;
125

  
126
			//for each reference
127
            while (rs.next()){
128
                
129
        		if ((i++ % modCount) == 0 && i!= 1 ){ logger.info(pluralString + " handled: " + (i-1));}
130
				
131
        		int collectionId = rs.getInt("CollectionId");
132
        		String name = rs.getString("Name");
133
        		String town = rs.getString("Town");
134
        		String ihCode = rs.getString("IHCode");
135
        		String subCollectionStr = rs.getString("Subcollection");
136
        		Integer tdwgArea = nullSafeInt(rs, "TDWGGazetteerFk");  //somehow redundant with town
137
        		String address = rs.getString("Address");           //only available for BGBM
138
        		Boolean cultCollFlag = rs.getBoolean("CultCollFlag");  //?? not really needed according to Henning
139
        		//TODO createdUpdates, NOtes		
140
      
141
        		try {
142
					
143
					//source ref
144
					Reference<?> sourceRef = state.getTransactionalSourceReference();
145
				
146
					
147
					Collection collection = Collection.NewInstance();
148
					collection.setName(name);
149
					if (isNotBlank("ihCode")){
150
						collection.setCode(ihCode);
151
						collection.setCodeStandard("Index Herbariorum");
152
					}
153
					
154
					collection.setTownOrLocation(town);
155
					collection.addSource(String.valueOf(collectionId), NAMESPACE_COLLECTION, sourceRef, null);
156
					
157
					collectionsToSave.add(collection);  //or subcollection ? 
158

  
159
					//subcollection
160
					if (isNotBlank(subCollectionStr)){
161
						Collection subCollection = Collection.NewInstance();
162
						subCollection.setName(subCollectionStr);
163
						subCollection.setSuperCollection(collection);
164
						collectionsToSave.add(subCollection);  //or subcollection ? 
165
						collection.addSource(String.valueOf(collectionId), NAMESPACE_SUBCOLLECTION, sourceRef, null);	
166
					}
167
					
168

  
169
				} catch (Exception e) {
170
					logger.warn("Exception in collection: CollectionId " + collectionId + ". " + e.getMessage());
171
//					e.printStackTrace();
172
				} 
173
                
174
            }
175
           
176
			logger.warn(pluralString + " to save: " + collectionsToSave.size());
177
			getCollectionService().save(collectionsToSave);	
178
			
179
			return success;
180
		} catch (SQLException e) {
181
			logger.error("SQLException:" +  e);
182
			return false;
183
		}
184
	}
185

  
186

  
187

  
188
	/* (non-Javadoc)
189
	 * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#getRelatedObjectsForPartition(java.sql.ResultSet)
190
	 */
191
	public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs) {
192
		//no related objects are needed
193
		Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<Object, Map<String, ? extends CdmBase>>();
194

  
195
		return result;
196
		
197
	}
198

  
199

  
200
	/* (non-Javadoc)
201
	 * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IoStateBase)
202
	 */
203
	@Override
204
	protected boolean doCheck(BerlinModelImportState state){
205
		IOValidator<BerlinModelImportState> validator = new AlgaTerraCollectionImportValidator();
206
		return validator.validate(state);
207
	}
208
	
209
	/* (non-Javadoc)
210
	 * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getTableName()
211
	 */
212
	@Override
213
	protected String getTableName() {
214
		return dbTableName;
215
	}
216
	
217
	/* (non-Javadoc)
218
	 * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getPluralString()
219
	 */
220
	@Override
221
	public String getPluralString() {
222
		return pluralString;
223
	}
224

  
225
	/* (non-Javadoc)
226
	 * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IImportConfigurator)
227
	 */
228
	protected boolean isIgnore(BerlinModelImportState bmState){
229
		AlgaTerraImportState state = (AlgaTerraImportState)bmState;
230
		return ! ( state.getAlgaTerraConfigurator().isDoSpecimen() ||  state.getAlgaTerraConfigurator().isDoTypes() );
231
	}
232
	
233
}
app-import/src/main/java/eu/etaxonomy/cdm/io/algaterra/AlgaTerraImportConfigurator.java
64 64
				, BerlinModelWebMarkerCategoryImport.class
65 65
				, BerlinModelWebMarkerImport.class
66 66
				
67
				, AlgaTerraCollectionImport.class
67 68
				, AlgaTerraSpecimenImport.class
68 69
				, AlgaTerraTypeImport.class
69 70
				
app-import/src/main/java/eu/etaxonomy/cdm/io/algaterra/AlgaTerraSpecimenImport.java
50 50
import eu.etaxonomy.cdm.model.description.TextData;
51 51
import eu.etaxonomy.cdm.model.name.BotanicalName;
52 52
import eu.etaxonomy.cdm.model.name.Rank;
53
import eu.etaxonomy.cdm.model.occurrence.Collection;
53 54
import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
54 55
import eu.etaxonomy.cdm.model.occurrence.FieldObservation;
55 56
import eu.etaxonomy.cdm.model.reference.Reference;
......
157 158
					DerivedUnitFacade facade = getDerivedUnit(state, ecoFactId, ecoFactMap, type);
158 159
					
159 160
					//field observation
160
					handleSingleSpecimen(rs, facade, state);
161
					handleSingleSpecimen(rs, facade, state, partitioner);
161 162
					
162 163
					handleEcoFactSpecificSpecimen(rs,facade, state);
163 164
					
......
465 466
			Set<String> taxonIdSet = new HashSet<String>();
466 467
			Set<String> fieldObservationIdSet = new HashSet<String>();
467 468
			Set<String> termsIdSet = new HashSet<String>();
469
			Set<String> collectionIdSet = new HashSet<String>();
468 470
			
469 471
			while (rs.next()){
470 472
				handleForeignKey(rs, taxonIdSet, "taxonId");
......
472 474
				handleForeignKey(rs, termsIdSet, "ClimateFk");
473 475
				handleForeignKey(rs, termsIdSet, "HabitatFk");
474 476
				handleForeignKey(rs, termsIdSet, "LifeFormFk");
477
				handleForeignKey(rs, collectionIdSet, "CollectionFk");
475 478
			}
476 479
			
477 480
			//taxon map
......
488 491
			Map<String, FieldObservation> fieldObservationMap = (Map<String, FieldObservation>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
489 492
			result.put(nameSpace, fieldObservationMap);
490 493

  
494
			//collections
495
			nameSpace = AlgaTerraCollectionImport.NAMESPACE_COLLECTION;
496
			cdmClass = Collection.class;
497
			idSet = collectionIdSet;
498
			Map<String, Collection> collectionMap = (Map<String, Collection>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
499
			result.put(nameSpace, collectionMap);
500

  
501
			//sub-collections
502
			nameSpace = AlgaTerraCollectionImport.NAMESPACE_SUBCOLLECTION;
503
			cdmClass = Collection.class;
504
			idSet = collectionIdSet;
505
			Map<String, Collection> subCollectionMap = (Map<String, Collection>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
506
			result.put(nameSpace, subCollectionMap);
507

  
491 508
			//terms
492 509
			nameSpace = AlgaTerraSpecimenImport.TERMS_NAMESPACE;
493 510
			cdmClass = FieldObservation.class;
app-import/src/main/java/eu/etaxonomy/cdm/io/algaterra/AlgaTerraSpecimenImportBase.java
13 13
import java.sql.Date;
14 14
import java.sql.ResultSet;
15 15
import java.sql.SQLException;
16
import java.util.Map;
16 17
import java.util.Set;
17 18
import java.util.UUID;
18 19

  
......
22 23

  
23 24
import eu.etaxonomy.cdm.api.facade.DerivedUnitFacade;
24 25
import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase;
26
import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelTaxonNameImport;
27
import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
25 28
import eu.etaxonomy.cdm.io.common.Source;
26 29
import eu.etaxonomy.cdm.model.agent.Team;
27 30
import eu.etaxonomy.cdm.model.common.OrderedTermVocabulary;
......
35 38
import eu.etaxonomy.cdm.model.location.ReferenceSystem;
36 39
import eu.etaxonomy.cdm.model.location.TdwgArea;
37 40
import eu.etaxonomy.cdm.model.location.WaterbodyOrCountry;
41
import eu.etaxonomy.cdm.model.name.TaxonNameBase;
42
import eu.etaxonomy.cdm.model.occurrence.Collection;
38 43

  
39 44
/**
40 45
 * @author a.mueller
......
148 153
		return "Locality";
149 154
	}
150 155
	
151
	protected void handleSingleSpecimen(ResultSet rs, DerivedUnitFacade facade, AlgaTerraImportState state) throws SQLException {
156
	protected void handleSingleSpecimen(ResultSet rs, DerivedUnitFacade facade, AlgaTerraImportState state, ResultSetPartitioner partitioner) throws SQLException {
152 157
		//FIXME missing fields #3084, #3085, #3080
153 158
		try {
154 159
			
......
167 172
			String collectorsNumber = rs.getString("CollectorsNumber");
168 173
			Date collectionDateStart = rs.getDate("CollectionDate");
169 174
			Date collectionDateEnd = rs.getDate("CollectionDateEnd");
170

  
175
			
176
			Integer collectionFk = nullSafeInt(rs,"CollectionFk");
177
			
178
			
171 179
			//location
172 180
			facade.setLocality(locality);
173 181
			    	
......
204 212
			   		facade.setDistanceToWaterSurface(intDepth);
205 213
			   	}
206 214
			}
207

  
208 215
			
209 216
			//field
210 217
			facade.setFieldNumber(collectorsNumber);
......
215 222
			//areas
216 223
			makeAreas(state, rs, facade);
217 224
			
225
			//collection
226
			if (collectionFk != null){
227
				Collection subCollection = state.getRelatedObject(AlgaTerraCollectionImport.NAMESPACE_SUBCOLLECTION, String.valueOf(collectionFk), Collection.class);
228
				if (subCollection != null){
229
					facade.setCollection(subCollection);
230
				}else{
231
					Collection collection = state.getRelatedObject(AlgaTerraCollectionImport.NAMESPACE_COLLECTION, String.valueOf(collectionFk), Collection.class);
232
					facade.setCollection(collection);
233
				}
234
			}
218 235
			
219 236
			//notes
220 237
			//TODO is this an annotation on field observation or on the derived unit?
app-import/src/main/java/eu/etaxonomy/cdm/io/algaterra/AlgaTerraTypeImport.java
36 36
import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignation;
37 37
import eu.etaxonomy.cdm.model.name.SpecimenTypeDesignationStatus;
38 38
import eu.etaxonomy.cdm.model.name.TaxonNameBase;
39
import eu.etaxonomy.cdm.model.occurrence.Collection;
39 40
import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
40 41
import eu.etaxonomy.cdm.model.occurrence.FieldObservation;
41 42
import eu.etaxonomy.cdm.model.reference.Reference;
......
151 152
					DerivedUnitFacade facade = getDerivedUnit(state, typeSpecimenId, typeSpecimenMap, type, ecoFactMap, ecoFactId);
152 153
					
153 154
					//field observation
154
					handleSingleSpecimen(rs, facade, state);
155
					handleSingleSpecimen(rs, facade, state, partitioner);
155 156
					
156 157
					handleTypeSpecimenSpecificSpecimen(rs,facade, state);
157 158
					
......
314 315
			Set<String> ecoFieldObservationIdSet = new HashSet<String>();
315 316
			Set<String> typeSpecimenIdSet = new HashSet<String>();
316 317
			Set<String> termsIdSet = new HashSet<String>();
318
			Set<String> collectionIdSet = new HashSet<String>();
317 319
			
318 320
			while (rs.next()){
319 321
				handleForeignKey(rs, nameIdSet, "nameFk");
320 322
				handleForeignKey(rs, ecoFieldObservationIdSet, "ecoFactFk");
321 323
				handleForeignKey(rs, typeSpecimenIdSet, "TypeSpecimenId");
324
				handleForeignKey(rs, collectionIdSet, "CollectionFk");
322 325
			}
323 326
			
324 327
			//name map
......
342 345
			Map<String, FieldObservation> typeSpecimenMap = (Map<String, FieldObservation>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
343 346
			result.put(nameSpace, typeSpecimenMap);
344 347

  
348

  
349
			//collections
350
			nameSpace = AlgaTerraCollectionImport.NAMESPACE_COLLECTION;
351
			cdmClass = Collection.class;
352
			idSet = collectionIdSet;
353
			Map<String, Collection> collectionMap = (Map<String, Collection>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
354
			result.put(nameSpace, collectionMap);
355

  
356
			//sub-collections
357
			nameSpace = AlgaTerraCollectionImport.NAMESPACE_SUBCOLLECTION;
358
			cdmClass = Collection.class;
359
			idSet = collectionIdSet;
360
			Map<String, Collection> subCollectionMap = (Map<String, Collection>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
361
			result.put(nameSpace, subCollectionMap);
362

  
345 363
			
346 364
			//
347 365
//			//terms

Also available in: Unified diff