/ - Diff - EDIT - EDIT Project Management

« Previous | Next »

Revision 3158fa66

Added by Andreas Müller almost 12 years ago

ID 3158fa66874cd1166e9609f7809daf7464c7ce98
Parent cbd8fc9c
Child 98c464e4

refactor PESI Description export to include NameDescriptions correctly

     import eu.etaxonomy.cdm.model.description.Feature;
     import eu.etaxonomy.cdm.model.description.IndividualsAssociation;
     import eu.etaxonomy.cdm.model.description.TaxonInteraction;
     import eu.etaxonomy.cdm.model.description.TaxonNameDescription;
     import eu.etaxonomy.cdm.model.description.TextData;
     import eu.etaxonomy.cdm.model.location.NamedArea;
     import eu.etaxonomy.cdm.model.location.TdwgArea;
-...
     			// Stores whether this invoke was successful or not.
     			boolean success = true;
     			// PESI: Clear the database table Note.
     //			doDelete(state);
     			// Get specific mappings: (CDM) DescriptionElement -> (PESI) Note
     			PesiExportMapping notesMapping = getNotesMapping();
     			notesMapping.initialize(state);
     			// Get specific mappings: (CDM) DescriptionElement -> (PESI) Occurrence
     			PesiExportMapping occurrenceMapping = getOccurrenceMapping();
     			occurrenceMapping.initialize(state);
     			// Get specific mappings: (CDM) DescriptionElement -> (PESI) Additional taxon source
     			PesiExportMapping addSourceSourceMapping = getAddTaxonSourceSourceMapping();
     			addSourceSourceMapping.initialize(state);
     			PesiExportMapping additionalSourceMapping = getAdditionalTaxonSourceMapping();
     			additionalSourceMapping.initialize(state);
     			// Get specific mappings: (CDM) DescriptionElement -> (PESI) Additional taxon source
     			PesiExportMapping vernacularMapping = getVernacularNamesMapping();
     			vernacularMapping.initialize(state);
     			// Get specific mappings: (CDM) DescriptionElement -> (PESI) Additional taxon source
     			PesiExportMapping imageMapping = getImageMapping();
     			imageMapping.initialize(state);
     			// Start transaction
     			success &= doPhase01(state);
     			success &= doPhase01(state, notesMapping, occurrenceMapping, addSourceSourceMapping, additionalSourceMapping, vernacularMapping, imageMapping);
     			// Start transaction
     			success &= doPhase01b(state, notesMapping, occurrenceMapping, addSourceSourceMapping, additionalSourceMapping, vernacularMapping, imageMapping);
     			logger.info("PHASE 2...");
-...
+    	}
     	//PHASE 01: Description Elements
     	private boolean doPhase01(PesiExportState state) throws SQLException {
     	private boolean doPhase01(PesiExportState state, PesiExportMapping notesMapping, PesiExportMapping occurrenceMapping, PesiExportMapping addSourceSourceMapping,
     			PesiExportMapping additionalSourceMapping, PesiExportMapping vernacularMapping, PesiExportMapping imageMapping) throws SQLException {
     		logger.info("PHASE 1...");
     		int count = 0;
     		int pastCount = 0;
     		boolean success = true;
     		int limit = state.getConfig().getLimitSave();
     		// Get specific mappings: (CDM) DescriptionElement -> (PESI) Note
     		PesiExportMapping notesMapping = getNotesMapping();
     		notesMapping.initialize(state);
     		// Get specific mappings: (CDM) DescriptionElement -> (PESI) Occurrence
     		PesiExportMapping occurrenceMapping = getOccurrenceMapping();
     		occurrenceMapping.initialize(state);
     		// Get specific mappings: (CDM) DescriptionElement -> (PESI) Additional taxon source
     		PesiExportMapping addSourceSourceMapping = getAddTaxonSourceSourceMapping();
     		addSourceSourceMapping.initialize(state);
     		PesiExportMapping additionalSourceMapping = getAdditionalTaxonSourceMapping();
     		additionalSourceMapping.initialize(state);
     		// Get specific mappings: (CDM) DescriptionElement -> (PESI) Additional taxon source
     		PesiExportMapping vernacularMapping = getVernacularNamesMapping();
     		vernacularMapping.initialize(state);
     		// Get specific mappings: (CDM) DescriptionElement -> (PESI) Additional taxon source
     		PesiExportMapping imageMapping = getImageMapping();
     		imageMapping.initialize(state);
     		List<Taxon> list = null;
     		List<Taxon> taxonList = null;
     		TransactionStatus txStatus = startTransaction(true);
     		logger.info("Started new transaction. Fetching some " + pluralString + " (max: " + limit + ") ...");
-...
     		ProfilerController.memorySnapshot();
     		//taxon descriptions
     		int partitionCount = 0;
     		while ((list = getNextTaxonPartition(Taxon.class, limit, partitionCount++, propPath )) != null   ) {
     		while ((taxonList = getNextTaxonPartition(Taxon.class, limit, partitionCount++, propPath )) != null   ) {
     			logger.info("Fetched " + list.size() + " " + pluralString + ". Exporting...");
     			logger.debug("Start snapshot, beginning of loop, fetched " + list.size() + " " + pluralString);
     			ProfilerController.memorySnapshot();
     			logger.info("Fetched " + taxonList.size() + " " + pluralString + ". Exporting...");
     			for (Taxon taxon : list) {
     			for (Taxon taxon : taxonList) {
     				countTaxa++;
     				doCount(count++, modCount, pluralString);
     				state.setCurrentTaxon(taxon);
     				success &= handleSingleTaxon(taxon, state, notesMapping, occurrenceMapping, addSourceSourceMapping,
     						additionalSourceMapping, vernacularMapping, imageMapping);
+    			}
     			list = null;
     			taxonList = null;
     			state.setCurrentTaxon(null);
     			// Commit transaction
-...
     			// Start transaction
     			txStatus = startTransaction(true);
     			logger.info("Started new transaction. Fetching some " + pluralString + " (max: " + limit + ") for description import ...");
     			logger.debug("Start snapshot, end of loop, fetched " + " " + pluralString);
     			ProfilerController.memorySnapshot();
+    		}
     //		//name descriptions
     //		while ((list = getNextNameDescriptionPartition( limit, partitionCount++, propPath )) != null   ) {
     //
     //			logger.info("Fetched " + list.size() + " " + pluralString + ". Exporting...");
     //
     //
     //			for (Taxon taxon : list) {
     //				countTaxa++;
     //				doCount(count++, modCount, pluralString);
     //				success &= handleSingleTaxon(taxon, state, notesMapping, occurrenceMapping, addittionalSourceMapping, vernacularMapping, imageMapping);
     //			}
     //			state.setCurrentTaxon(null);
     //
     //			// Commit transaction
     //			commitTransaction(txStatus);
     //			logger.info("Exported " + (count - pastCount) + " " + pluralString + ". Total: " + count);
     //			pastCount = count;
     //
     //			// Start transaction
     //			txStatus = startTransaction(true);
     //			logger.info("Started new transaction. Fetching some " + pluralString + " (max: " + limit + ") for description import ...");
     //		}
     		logger.info("No " + pluralString + " left to fetch.");
     		logger.info("Partition: " + partitionCount);
     		logger.info("Taxa: " + countTaxa);
     		logger.info("Desc: " + countDescriptions);
     		logger.info("Distr: " + countDistribution);
     		logger.info("Occur: " + countOccurrence);
     		logger.info("Commons: " + countCommonName);
     		logger.info("AddSrc: " + countAdditionalSources);
     		logger.info("Images: " + countImages);
     		logger.info("Notes: " + countNotes);
     		logger.info("Others: " + countOthers);
     		// Commit transaction
     		commitTransaction(txStatus);
     		logger.debug("Committed transaction.");
     		return success;
+    	}
     	//PHASE 01b: Name Descriptions
     	private boolean doPhase01b(PesiExportState state, PesiExportMapping notesMapping, PesiExportMapping occurrenceMapping, PesiExportMapping addSourceSourceMapping,
     			PesiExportMapping additionalSourceMapping, PesiExportMapping vernacularMapping, PesiExportMapping imageMapping) throws SQLException {
     		logger.info("PHASE 1b...");
     		int count = 0;
     		int pastCount = 0;
     		boolean success = true;
     		int limit = state.getConfig().getLimitSave();
     		List<TaxonNameDescription> nameDescList = null;
     		TransactionStatus txStatus = startTransaction(true);
     		logger.info("Started new transaction. Fetching some name descriptions (max: " + limit + ") ...");
     		List<String> propPath = Arrays.asList(new String[]{"descriptions.elements.*"});
     		if (list == null) {
     			logger.info("No " + pluralString + " left to fetch.");
     			logger.info("Partition: " + partitionCount);
     			logger.info("Taxa: " + countTaxa);
     			logger.info("Desc: " + countDescriptions);
     			logger.info("Distr: " + countDistribution);
     			logger.info("Occur: " + countOccurrence);
     			logger.info("Commons: " + countCommonName);
     			logger.info("AddSrc: " + countAdditionalSources);
     			logger.info("Images: " + countImages);
     			logger.info("Notes: " + countNotes);
     			logger.info("Others: " + countOthers);
     		//name descriptions
     		int partitionCount = 0;
     		while ((nameDescList = getNextNameDescriptionPartition( limit, partitionCount++, propPath )) != null   ) {
     			logger.info("Fetched " + nameDescList.size() + " name descriptions. Exporting...");
     			for (TaxonNameDescription desc : nameDescList) {
     				countTaxa++;
     				doCount(count++, modCount, "name descriptions");
     				boolean isImageGallery = desc.isImageGallery();
     				TaxonNameBase<?,?> name = desc.getTaxonName();
     				for (DescriptionElementBase element : desc.getElements()){
     					if (isPurePesiName(name)){
     						success &= handleDescriptionElement(state, notesMapping, occurrenceMapping, vernacularMapping, imageMapping,
     								addSourceSourceMapping, additionalSourceMapping, isImageGallery, element);
     					}else{
     						for (TaxonBase<?> taxonBase : name.getTaxonBases()){
     							if (isPesiTaxon(taxonBase)){
     								state.setCurrentTaxon(taxonBase);
     								success &= handleDescriptionElement(state, notesMapping, occurrenceMapping, vernacularMapping, imageMapping,
     										addSourceSourceMapping, additionalSourceMapping, isImageGallery, element);
     								state.setSourceForAdditionalSourceCreated(true);
+    							}
+    						}
     						state.setSourceForAdditionalSourceCreated(false);
+    					}
+    				}
+    			}
     			nameDescList = null;
     			state.setCurrentTaxon(null);
     			// Commit transaction
     			commitTransaction(txStatus);
     			logger.info("Exported " + (count - pastCount) + " name descriptions. Total: " + count);
     			pastCount = count;
     			// Start transaction
     			txStatus = startTransaction(true);
     			logger.info("Started new transaction. Fetching some name descriptions (max: " + limit + ") for description import ...");
+    		}
     		list = null;
     		logger.info("No " + pluralString + " left to fetch.");
     		logger.info("Partition: " + partitionCount);
     		logger.info("Taxa: " + countTaxa);
     		logger.info("Desc: " + countDescriptions);
     		logger.info("Distr: " + countDistribution);
     		logger.info("Occur: " + countOccurrence);
     		logger.info("Commons: " + countCommonName);
     		logger.info("AddSrc: " + countAdditionalSources);
     		logger.info("Images: " + countImages);
     		logger.info("Notes: " + countNotes);
     		logger.info("Others: " + countOthers);
     		// Commit transaction
     		commitTransaction(txStatus);
     		logger.debug("Committed transaction.");
-...
     		Set<DescriptionBase<?>> descriptions = new HashSet<DescriptionBase<?>>();
     		descriptions.addAll(taxon.getDescriptions());
     		//FIXME incorrect as this creates duplicates
     		descriptions.addAll(taxon.getName().getDescriptions());
     		state.setCurrentTaxon(taxon);
     		for (DescriptionBase<?> desc : descriptions){
     			countDescriptions++;
-...
+    				}
     			}else if (isAdditionalTaxonSource(element)){
     				countAdditionalSources++;
     				success &= addSourceSourceMapping.invoke(element);
     				if (! state.isSourceForAdditionalSourceCreated()){
     					success &= addSourceSourceMapping.invoke(element);
+    				}
     				success &= additionalSourceMapping.invoke(element);
     			}else if (isExcludedNote(element)){
     				//do nothing
-...
     		} else
     			try {
     				if (state.getTransformer().getKeyByNamedArea(area) == null){
     					logger.warn("Area not available in PESI transformer " +  area.getTitleCache() + ", " + area.getRepresentation(Language.ENGLISH()).getAbbreviatedLabel());
     					String warning = "Area (%s,%s) not available in PESI transformer for taxon %S: ";
     					TaxonBase<?> taxon =  state.getCurrentTaxon();
     					warning = String.format(warning, area.getTitleCache(), area.getRepresentation(Language.ENGLISH()).getAbbreviatedLabel(),taxon ==null? "-" : taxon.getTitleCache());
     					logger.warn(warning);
     					return false;
+    				}
     			} catch (UndefinedTransformerMethodException e1) {
-...
     	 */
     	@SuppressWarnings("unused")  //used by mapper
     	private static Integer getTaxonFk(DescriptionElementBase deb, PesiExportState state) {
     		IdentifiableEntity<?> entity = state.getCurrentTaxon();
     		TaxonBase<?> entity = state.getCurrentTaxon();
     		return state.getDbId(entity);
+    	}
-...
     	@SuppressWarnings("unused")
     	private static String getTaxonFullNameCache(DescriptionElementBase deb, PesiExportState state) {
     		IdentifiableEntity<?> taxon = state.getCurrentTaxon();
     		TaxonBase<?> taxonDeproxy = CdmBase.deproxy(taxon, TaxonBase.class);
     		TaxonNameBase<?,?> taxonName = taxonDeproxy.getName();
     		taxonDeproxy = null;
     		TaxonBase<?> taxon =  state.getCurrentTaxon();
     		TaxonNameBase<?,?> taxonName = taxon.getName();
     		NonViralName<?> nvn = CdmBase.deproxy(taxonName, NonViralName.class);
     		String result = getCacheStrategy(nvn).getTitleCache(nvn);
     		nvn = null;
     		return result;
+    	}
-...
     		mapping.addMapper(DbConstantMapper.NewInstance("SourceUseFk", Types.INTEGER, PesiTransformer.NOMENCLATURAL_REFERENCE));
     		mapping.addMapper(DbConstantMapper.NewInstance("SourceUseCache", Types.VARCHAR, PesiTransformer.STR_NOMENCLATURAL_REFERENCE));
     //		mapping.addMapper(DbIgnoreMapper.NewInstance("SourceDetail", "SourceDetails not available for additional sources"));
     		mapping.addMapper(DbExportIgnoreMapper.NewInstance("SourceDetail", "SourceDetails not available for additional sources"));
     		return mapping;
+    	}

     	private boolean isPesiNameDescriptionTaxon(TaxonNameDescription nameDescription) {
     		TaxonNameBase name = nameDescription.getTaxonName();
     		TaxonNameBase<?,?> name = nameDescription.getTaxonName();
     		if (isPurePesiName(name)){
     			return true;
     		}else{
     			Set<TaxonBase> taxa = name.getTaxonBases();
     			for (TaxonBase taxonBase : taxa){
     			for (TaxonBase<?> taxonBase : taxa){
     				if (isPesiTaxon(taxonBase)){
     					return true;
+    				}
-...
+    	}
     	/**
     	 * @see #isPesiTaxon(TaxonBase, boolean)
     	 * @param taxonBase
     	 * @return
     	 */
     	protected static boolean isPesiTaxon(TaxonBase taxonBase) {
     		return isPesiTaxon(taxonBase, false);
+    	}

     import eu.etaxonomy.cdm.model.common.CdmBase;
     import eu.etaxonomy.cdm.model.common.IdentifiableEntity;
     import eu.etaxonomy.cdm.model.common.MarkerType;
     import eu.etaxonomy.cdm.model.taxon.TaxonBase;
     /**
      * The export state class.
-...
     	private IdentifiableEntity<?> currentToObject;
     	private IdentifiableEntity<?> currentFromObject;
     	private IdentifiableEntity<?> currentTaxon;
     	private TaxonBase<?> currentTaxon;
     	private boolean sourceForAdditionalSourceCreated = false;
     	private Map<UUID, MarkerType> markerTypeMap = new HashMap<UUID, MarkerType>();
     	public static final UUID uuidUserDefinedMarkerTypeVocabulary = UUID.fromString("5f02a261-fd7d-4fce-bbe4-21472de8cd51");
-...
+    	}
     	public IdentifiableEntity<?> getCurrentTaxon() {
     	public TaxonBase<?> getCurrentTaxon() {
     		return currentTaxon;
+    	}
     	public void setCurrentTaxon(IdentifiableEntity<?> currentTaxon) {
     	public void setCurrentTaxon(TaxonBase<?> currentTaxon) {
     		this.currentTaxon = currentTaxon;
+    	}
-...
+    	}
     	public boolean isSourceForAdditionalSourceCreated() {
     		return sourceForAdditionalSourceCreated;
+    	}
     	public void setSourceForAdditionalSourceCreated(
     			boolean sourceForAdditionalSourceCreated) {
     		this.sourceForAdditionalSourceCreated = sourceForAdditionalSourceCreated;
+    	}
+    }

Also available in: Unified diff

Project

General

Profile

EDIT

Revision 3158fa66

Added by Andreas Müller almost 12 years ago