Revision 3158fa66
Added by Andreas Müller almost 12 years ago
cdm-pesi/src/main/java/eu/etaxonomy/cdm/io/pesi/out/PesiDescriptionExport.java | ||
---|---|---|
63 | 63 |
import eu.etaxonomy.cdm.model.description.Feature; |
64 | 64 |
import eu.etaxonomy.cdm.model.description.IndividualsAssociation; |
65 | 65 |
import eu.etaxonomy.cdm.model.description.TaxonInteraction; |
66 |
import eu.etaxonomy.cdm.model.description.TaxonNameDescription; |
|
66 | 67 |
import eu.etaxonomy.cdm.model.description.TextData; |
67 | 68 |
import eu.etaxonomy.cdm.model.location.NamedArea; |
68 | 69 |
import eu.etaxonomy.cdm.model.location.TdwgArea; |
... | ... | |
144 | 145 |
// Stores whether this invoke was successful or not. |
145 | 146 |
boolean success = true; |
146 | 147 |
|
147 |
// PESI: Clear the database table Note. |
|
148 |
// doDelete(state); |
|
149 |
|
|
148 |
// Get specific mappings: (CDM) DescriptionElement -> (PESI) Note |
|
149 |
PesiExportMapping notesMapping = getNotesMapping(); |
|
150 |
notesMapping.initialize(state); |
|
151 |
|
|
152 |
// Get specific mappings: (CDM) DescriptionElement -> (PESI) Occurrence |
|
153 |
PesiExportMapping occurrenceMapping = getOccurrenceMapping(); |
|
154 |
occurrenceMapping.initialize(state); |
|
155 |
|
|
156 |
// Get specific mappings: (CDM) DescriptionElement -> (PESI) Additional taxon source |
|
157 |
PesiExportMapping addSourceSourceMapping = getAddTaxonSourceSourceMapping(); |
|
158 |
addSourceSourceMapping.initialize(state); |
|
159 |
PesiExportMapping additionalSourceMapping = getAdditionalTaxonSourceMapping(); |
|
160 |
additionalSourceMapping.initialize(state); |
|
161 |
|
|
162 |
// Get specific mappings: (CDM) DescriptionElement -> (PESI) Additional taxon source |
|
163 |
PesiExportMapping vernacularMapping = getVernacularNamesMapping(); |
|
164 |
vernacularMapping.initialize(state); |
|
165 |
|
|
166 |
// Get specific mappings: (CDM) DescriptionElement -> (PESI) Additional taxon source |
|
167 |
PesiExportMapping imageMapping = getImageMapping(); |
|
168 |
imageMapping.initialize(state); |
|
169 |
|
|
150 | 170 |
// Start transaction |
151 |
success &= doPhase01(state); |
|
171 |
success &= doPhase01(state, notesMapping, occurrenceMapping, addSourceSourceMapping, additionalSourceMapping, vernacularMapping, imageMapping); |
|
172 |
|
|
173 |
// Start transaction |
|
174 |
success &= doPhase01b(state, notesMapping, occurrenceMapping, addSourceSourceMapping, additionalSourceMapping, vernacularMapping, imageMapping); |
|
152 | 175 |
|
153 | 176 |
|
154 | 177 |
logger.info("PHASE 2..."); |
... | ... | |
169 | 192 |
} |
170 | 193 |
|
171 | 194 |
//PHASE 01: Description Elements |
172 |
private boolean doPhase01(PesiExportState state) throws SQLException { |
|
195 |
private boolean doPhase01(PesiExportState state, PesiExportMapping notesMapping, PesiExportMapping occurrenceMapping, PesiExportMapping addSourceSourceMapping, |
|
196 |
PesiExportMapping additionalSourceMapping, PesiExportMapping vernacularMapping, PesiExportMapping imageMapping) throws SQLException { |
|
173 | 197 |
logger.info("PHASE 1..."); |
174 | 198 |
int count = 0; |
175 | 199 |
int pastCount = 0; |
176 | 200 |
boolean success = true; |
177 | 201 |
int limit = state.getConfig().getLimitSave(); |
178 | 202 |
|
179 |
// Get specific mappings: (CDM) DescriptionElement -> (PESI) Note |
|
180 |
PesiExportMapping notesMapping = getNotesMapping(); |
|
181 |
notesMapping.initialize(state); |
|
182 |
|
|
183 |
// Get specific mappings: (CDM) DescriptionElement -> (PESI) Occurrence |
|
184 |
PesiExportMapping occurrenceMapping = getOccurrenceMapping(); |
|
185 |
occurrenceMapping.initialize(state); |
|
186 |
|
|
187 |
// Get specific mappings: (CDM) DescriptionElement -> (PESI) Additional taxon source |
|
188 |
PesiExportMapping addSourceSourceMapping = getAddTaxonSourceSourceMapping(); |
|
189 |
addSourceSourceMapping.initialize(state); |
|
190 |
PesiExportMapping additionalSourceMapping = getAdditionalTaxonSourceMapping(); |
|
191 |
additionalSourceMapping.initialize(state); |
|
192 |
|
|
193 |
// Get specific mappings: (CDM) DescriptionElement -> (PESI) Additional taxon source |
|
194 |
PesiExportMapping vernacularMapping = getVernacularNamesMapping(); |
|
195 |
vernacularMapping.initialize(state); |
|
196 |
|
|
197 |
// Get specific mappings: (CDM) DescriptionElement -> (PESI) Additional taxon source |
|
198 |
PesiExportMapping imageMapping = getImageMapping(); |
|
199 |
imageMapping.initialize(state); |
|
200 |
|
|
201 |
|
|
202 |
List<Taxon> list = null; |
|
203 |
List<Taxon> taxonList = null; |
|
203 | 204 |
|
204 | 205 |
TransactionStatus txStatus = startTransaction(true); |
205 | 206 |
logger.info("Started new transaction. Fetching some " + pluralString + " (max: " + limit + ") ..."); |
... | ... | |
209 | 210 |
ProfilerController.memorySnapshot(); |
210 | 211 |
//taxon descriptions |
211 | 212 |
int partitionCount = 0; |
212 |
while ((list = getNextTaxonPartition(Taxon.class, limit, partitionCount++, propPath )) != null ) {
|
|
213 |
while ((taxonList = getNextTaxonPartition(Taxon.class, limit, partitionCount++, propPath )) != null ) {
|
|
213 | 214 |
|
214 |
logger.info("Fetched " + list.size() + " " + pluralString + ". Exporting..."); |
|
215 |
|
|
216 |
logger.debug("Start snapshot, beginning of loop, fetched " + list.size() + " " + pluralString); |
|
217 |
ProfilerController.memorySnapshot(); |
|
215 |
logger.info("Fetched " + taxonList.size() + " " + pluralString + ". Exporting..."); |
|
218 | 216 |
|
219 |
for (Taxon taxon : list) {
|
|
217 |
for (Taxon taxon : taxonList) {
|
|
220 | 218 |
countTaxa++; |
221 | 219 |
doCount(count++, modCount, pluralString); |
220 |
state.setCurrentTaxon(taxon); |
|
222 | 221 |
success &= handleSingleTaxon(taxon, state, notesMapping, occurrenceMapping, addSourceSourceMapping, |
223 | 222 |
additionalSourceMapping, vernacularMapping, imageMapping); |
224 | 223 |
} |
225 |
list = null;
|
|
224 |
taxonList = null;
|
|
226 | 225 |
state.setCurrentTaxon(null); |
227 | 226 |
|
228 | 227 |
// Commit transaction |
... | ... | |
233 | 232 |
// Start transaction |
234 | 233 |
txStatus = startTransaction(true); |
235 | 234 |
logger.info("Started new transaction. Fetching some " + pluralString + " (max: " + limit + ") for description import ..."); |
236 |
logger.debug("Start snapshot, end of loop, fetched " + " " + pluralString); |
|
237 |
ProfilerController.memorySnapshot(); |
|
238 | 235 |
} |
239 | 236 |
|
240 |
// //name descriptions |
|
241 |
// while ((list = getNextNameDescriptionPartition( limit, partitionCount++, propPath )) != null ) { |
|
242 |
// |
|
243 |
// logger.info("Fetched " + list.size() + " " + pluralString + ". Exporting..."); |
|
244 |
// |
|
245 |
// |
|
246 |
// for (Taxon taxon : list) { |
|
247 |
// countTaxa++; |
|
248 |
// doCount(count++, modCount, pluralString); |
|
249 |
// success &= handleSingleTaxon(taxon, state, notesMapping, occurrenceMapping, addittionalSourceMapping, vernacularMapping, imageMapping); |
|
250 |
// } |
|
251 |
// state.setCurrentTaxon(null); |
|
252 |
// |
|
253 |
// // Commit transaction |
|
254 |
// commitTransaction(txStatus); |
|
255 |
// logger.info("Exported " + (count - pastCount) + " " + pluralString + ". Total: " + count); |
|
256 |
// pastCount = count; |
|
257 |
// |
|
258 |
// // Start transaction |
|
259 |
// txStatus = startTransaction(true); |
|
260 |
// logger.info("Started new transaction. Fetching some " + pluralString + " (max: " + limit + ") for description import ..."); |
|
261 |
// } |
|
262 |
|
|
237 |
|
|
238 |
logger.info("No " + pluralString + " left to fetch."); |
|
239 |
logger.info("Partition: " + partitionCount); |
|
240 |
logger.info("Taxa: " + countTaxa); |
|
241 |
logger.info("Desc: " + countDescriptions); |
|
242 |
logger.info("Distr: " + countDistribution); |
|
243 |
logger.info("Occur: " + countOccurrence); |
|
244 |
logger.info("Commons: " + countCommonName); |
|
245 |
logger.info("AddSrc: " + countAdditionalSources); |
|
246 |
logger.info("Images: " + countImages); |
|
247 |
logger.info("Notes: " + countNotes); |
|
248 |
logger.info("Others: " + countOthers); |
|
263 | 249 |
|
250 |
// Commit transaction |
|
251 |
commitTransaction(txStatus); |
|
252 |
logger.debug("Committed transaction."); |
|
253 |
return success; |
|
254 |
} |
|
255 |
|
|
256 |
//PHASE 01b: Name Descriptions |
|
257 |
private boolean doPhase01b(PesiExportState state, PesiExportMapping notesMapping, PesiExportMapping occurrenceMapping, PesiExportMapping addSourceSourceMapping, |
|
258 |
PesiExportMapping additionalSourceMapping, PesiExportMapping vernacularMapping, PesiExportMapping imageMapping) throws SQLException { |
|
259 |
logger.info("PHASE 1b..."); |
|
260 |
int count = 0; |
|
261 |
int pastCount = 0; |
|
262 |
boolean success = true; |
|
263 |
int limit = state.getConfig().getLimitSave(); |
|
264 |
|
|
265 |
List<TaxonNameDescription> nameDescList = null; |
|
266 |
|
|
267 |
TransactionStatus txStatus = startTransaction(true); |
|
268 |
logger.info("Started new transaction. Fetching some name descriptions (max: " + limit + ") ..."); |
|
269 |
List<String> propPath = Arrays.asList(new String[]{"descriptions.elements.*"}); |
|
264 | 270 |
|
265 |
if (list == null) { |
|
266 |
logger.info("No " + pluralString + " left to fetch."); |
|
267 |
logger.info("Partition: " + partitionCount); |
|
268 |
logger.info("Taxa: " + countTaxa); |
|
269 |
logger.info("Desc: " + countDescriptions); |
|
270 |
logger.info("Distr: " + countDistribution); |
|
271 |
logger.info("Occur: " + countOccurrence); |
|
272 |
logger.info("Commons: " + countCommonName); |
|
273 |
logger.info("AddSrc: " + countAdditionalSources); |
|
274 |
logger.info("Images: " + countImages); |
|
275 |
logger.info("Notes: " + countNotes); |
|
276 |
logger.info("Others: " + countOthers); |
|
271 |
//name descriptions |
|
272 |
int partitionCount = 0; |
|
273 |
while ((nameDescList = getNextNameDescriptionPartition( limit, partitionCount++, propPath )) != null ) { |
|
274 |
|
|
275 |
logger.info("Fetched " + nameDescList.size() + " name descriptions. Exporting..."); |
|
276 |
|
|
277 |
for (TaxonNameDescription desc : nameDescList) { |
|
278 |
countTaxa++; |
|
279 |
doCount(count++, modCount, "name descriptions"); |
|
280 |
boolean isImageGallery = desc.isImageGallery(); |
|
281 |
|
|
282 |
TaxonNameBase<?,?> name = desc.getTaxonName(); |
|
283 |
|
|
284 |
for (DescriptionElementBase element : desc.getElements()){ |
|
285 |
if (isPurePesiName(name)){ |
|
286 |
success &= handleDescriptionElement(state, notesMapping, occurrenceMapping, vernacularMapping, imageMapping, |
|
287 |
addSourceSourceMapping, additionalSourceMapping, isImageGallery, element); |
|
288 |
}else{ |
|
289 |
for (TaxonBase<?> taxonBase : name.getTaxonBases()){ |
|
290 |
if (isPesiTaxon(taxonBase)){ |
|
291 |
state.setCurrentTaxon(taxonBase); |
|
292 |
success &= handleDescriptionElement(state, notesMapping, occurrenceMapping, vernacularMapping, imageMapping, |
|
293 |
addSourceSourceMapping, additionalSourceMapping, isImageGallery, element); |
|
294 |
state.setSourceForAdditionalSourceCreated(true); |
|
295 |
} |
|
296 |
} |
|
297 |
state.setSourceForAdditionalSourceCreated(false); |
|
298 |
} |
|
299 |
} |
|
300 |
} |
|
301 |
nameDescList = null; |
|
302 |
state.setCurrentTaxon(null); |
|
303 |
|
|
304 |
// Commit transaction |
|
305 |
commitTransaction(txStatus); |
|
306 |
logger.info("Exported " + (count - pastCount) + " name descriptions. Total: " + count); |
|
307 |
pastCount = count; |
|
277 | 308 |
|
309 |
// Start transaction |
|
310 |
txStatus = startTransaction(true); |
|
311 |
logger.info("Started new transaction. Fetching some name descriptions (max: " + limit + ") for description import ..."); |
|
278 | 312 |
} |
279 | 313 |
|
280 |
list = null; |
|
314 |
logger.info("No " + pluralString + " left to fetch."); |
|
315 |
logger.info("Partition: " + partitionCount); |
|
316 |
logger.info("Taxa: " + countTaxa); |
|
317 |
logger.info("Desc: " + countDescriptions); |
|
318 |
logger.info("Distr: " + countDistribution); |
|
319 |
logger.info("Occur: " + countOccurrence); |
|
320 |
logger.info("Commons: " + countCommonName); |
|
321 |
logger.info("AddSrc: " + countAdditionalSources); |
|
322 |
logger.info("Images: " + countImages); |
|
323 |
logger.info("Notes: " + countNotes); |
|
324 |
logger.info("Others: " + countOthers); |
|
325 |
|
|
281 | 326 |
// Commit transaction |
282 | 327 |
commitTransaction(txStatus); |
283 | 328 |
logger.debug("Committed transaction."); |
... | ... | |
291 | 336 |
Set<DescriptionBase<?>> descriptions = new HashSet<DescriptionBase<?>>(); |
292 | 337 |
descriptions.addAll(taxon.getDescriptions()); |
293 | 338 |
|
294 |
//FIXME incorrect as this creates duplicates |
|
295 |
descriptions.addAll(taxon.getName().getDescriptions()); |
|
296 |
|
|
297 |
state.setCurrentTaxon(taxon); |
|
298 | 339 |
for (DescriptionBase<?> desc : descriptions){ |
299 | 340 |
countDescriptions++; |
300 | 341 |
|
... | ... | |
335 | 376 |
} |
336 | 377 |
}else if (isAdditionalTaxonSource(element)){ |
337 | 378 |
countAdditionalSources++; |
338 |
success &= addSourceSourceMapping.invoke(element); |
|
379 |
if (! state.isSourceForAdditionalSourceCreated()){ |
|
380 |
success &= addSourceSourceMapping.invoke(element); |
|
381 |
} |
|
339 | 382 |
success &= additionalSourceMapping.invoke(element); |
340 | 383 |
}else if (isExcludedNote(element)){ |
341 | 384 |
//do nothing |
... | ... | |
389 | 432 |
} else |
390 | 433 |
try { |
391 | 434 |
if (state.getTransformer().getKeyByNamedArea(area) == null){ |
392 |
logger.warn("Area not available in PESI transformer " + area.getTitleCache() + ", " + area.getRepresentation(Language.ENGLISH()).getAbbreviatedLabel()); |
|
435 |
String warning = "Area (%s,%s) not available in PESI transformer for taxon %S: "; |
|
436 |
TaxonBase<?> taxon = state.getCurrentTaxon(); |
|
437 |
warning = String.format(warning, area.getTitleCache(), area.getRepresentation(Language.ENGLISH()).getAbbreviatedLabel(),taxon ==null? "-" : taxon.getTitleCache()); |
|
438 |
logger.warn(warning); |
|
393 | 439 |
return false; |
394 | 440 |
} |
395 | 441 |
} catch (UndefinedTransformerMethodException e1) { |
... | ... | |
754 | 800 |
*/ |
755 | 801 |
@SuppressWarnings("unused") //used by mapper |
756 | 802 |
private static Integer getTaxonFk(DescriptionElementBase deb, PesiExportState state) { |
757 |
IdentifiableEntity<?> entity = state.getCurrentTaxon();
|
|
803 |
TaxonBase<?> entity = state.getCurrentTaxon();
|
|
758 | 804 |
return state.getDbId(entity); |
759 | 805 |
} |
760 | 806 |
|
... | ... | |
777 | 823 |
@SuppressWarnings("unused") |
778 | 824 |
private static String getTaxonFullNameCache(DescriptionElementBase deb, PesiExportState state) { |
779 | 825 |
|
780 |
IdentifiableEntity<?> taxon = state.getCurrentTaxon(); |
|
781 |
TaxonBase<?> taxonDeproxy = CdmBase.deproxy(taxon, TaxonBase.class); |
|
782 |
TaxonNameBase<?,?> taxonName = taxonDeproxy.getName(); |
|
783 |
taxonDeproxy = null; |
|
826 |
TaxonBase<?> taxon = state.getCurrentTaxon(); |
|
827 |
TaxonNameBase<?,?> taxonName = taxon.getName(); |
|
784 | 828 |
NonViralName<?> nvn = CdmBase.deproxy(taxonName, NonViralName.class); |
785 | 829 |
String result = getCacheStrategy(nvn).getTitleCache(nvn); |
786 |
nvn = null; |
|
787 | 830 |
return result; |
788 | 831 |
} |
789 | 832 |
|
... | ... | |
905 | 948 |
mapping.addMapper(DbConstantMapper.NewInstance("SourceUseFk", Types.INTEGER, PesiTransformer.NOMENCLATURAL_REFERENCE)); |
906 | 949 |
mapping.addMapper(DbConstantMapper.NewInstance("SourceUseCache", Types.VARCHAR, PesiTransformer.STR_NOMENCLATURAL_REFERENCE)); |
907 | 950 |
|
908 |
// mapping.addMapper(DbIgnoreMapper.NewInstance("SourceDetail", "SourceDetails not available for additional sources"));
|
|
951 |
mapping.addMapper(DbExportIgnoreMapper.NewInstance("SourceDetail", "SourceDetails not available for additional sources"));
|
|
909 | 952 |
|
910 | 953 |
return mapping; |
911 | 954 |
} |
Also available in: Unified diff
refactor PESI Description export to include NameDescriptions correctly