Project

General

Profile

« Previous | Next » 

Revision d9f92b40

Added by Andreas Müller over 3 years ago

ref #8509, ref #8508 some fixes to E+M PESI export and validation

View differences:

cdm-pesi/src/main/java/eu/etaxonomy/cdm/io/pesi/out/PesiSourceExport.java
425 425
		mapping.addMapper(DbDoiMapper.NewInstance("doi", "Doi"));
426 426

  
427 427
		mapping.addMapper(MethodMapper.NewInstance("NomRefCache", this));
428
		logger.warn("URI mapping needs to be combined");
429
//		mapping.addMapper(DbUriMapper.NewInstance("uri", "Link"));
428 430
		mapping.addMapper(DbExtensionMapper.NewInstance(ExtensionType.URL(), "Link"));
429 431
		mapping.addMapper(DbAnnotationMapper.NewInstance((String)null, "Notes"));
430 432
		mapping.addMapper(MethodMapper.NewInstance("RefIdInSource", this));
cdm-pesi/src/main/java/eu/etaxonomy/cdm/io/pesi/out/PesiTaxonExport.java
26 26

  
27 27
import org.apache.commons.lang.StringUtils;
28 28
import org.apache.log4j.Logger;
29
import org.joda.time.DateTime;
30
import org.joda.time.format.DateTimeFormat;
31
import org.joda.time.format.DateTimeFormatter;
32 29
import org.springframework.stereotype.Component;
33 30
import org.springframework.transaction.TransactionStatus;
34 31

  
......
37 34
import eu.etaxonomy.cdm.io.common.Source;
38 35
import eu.etaxonomy.cdm.io.common.mapping.UndefinedTransformerMethodException;
39 36
import eu.etaxonomy.cdm.io.common.mapping.out.DbConstantMapper;
40
import eu.etaxonomy.cdm.io.common.mapping.out.DbExtensionMapper;
41 37
import eu.etaxonomy.cdm.io.common.mapping.out.DbLastActionMapper;
42 38
import eu.etaxonomy.cdm.io.common.mapping.out.DbObjectMapper;
43 39
import eu.etaxonomy.cdm.io.common.mapping.out.DbStringMapper;
......
172 168
			success &= doPhase01(state, mapping, additionalSourceMapping);
173 169

  
174 170
			//"PHASE 1b: Handle names without taxa ...
175
			success &= doNames(state, additionalSourceMapping);
171
			success &= doPhase01b_Names(state, additionalSourceMapping);
176 172

  
177 173
			// 2nd Round: Add ParentTaxonFk to each taxon
178 174
			success &= doPhase02(state);
......
881 877
		return inferredSynonymsDataToBeSaved;
882 878
	}
883 879

  
884

  
885 880
	/**
886 881
	 * Handles names that do not appear in taxa
887 882
	 * @param state
888 883
	 * @param mapping
889 884
	 */
890
	private boolean doNames(PesiExportState state, PesiExportMapping additionalSourceMapping)  throws SQLException {
885
	private boolean doPhase01b_Names(PesiExportState state, PesiExportMapping additionalSourceMapping) {
891 886

  
892 887
		boolean success = true;
893 888
		if (! state.getConfig().isDoPureNames()){
......
1399 1394
			    //TODO define for FE + IF and for multiple sources
1400 1395
			    result = cacheStrategy.getFullTitleCache(taxonName, tagRules);
1401 1396
			}
1402
			return result.replaceAll(",?\\<@status@\\>.*\\</@status@\\>", "");
1397
			return result.replaceAll("(, ?)?\\<@status@\\>.*\\</@status@\\>", "").trim();
1403 1398
		}
1404 1399
	}
1405 1400

  
......
1599 1594
		Integer result = null;
1600 1595

  
1601 1596
		try {
1602
			if (isMisappliedName(taxon)) {
1603
				Synonym synonym = Synonym.NewInstance(null, null);
1604

  
1605
				// This works as long as only the instance is important to differentiate between TaxonStatus.
1606
				result = PesiTransformer.taxonBase2statusFk(synonym); // Auct References are treated as Synonyms in Datawarehouse now.
1607
			} else {
1597
//			if (isMisappliedName(taxon)) {
1598
//				Synonym synonym = Synonym.NewInstance(null, null);
1599
//
1600
//				// This works as long as only the instance is important to differentiate between TaxonStatus.
1601
//				result = PesiTransformer.taxonBase2statusFk(synonym); // Auct References are treated as Synonyms in datawarehouse now.
1602
//			} else {
1603
		    //this should work now, the method itself distinguishes MAN etc.
1608 1604
				result = PesiTransformer.taxonBase2statusFk(taxon);
1609
			}
1605
//			}
1610 1606
		} catch (Exception e) {
1611 1607
			e.printStackTrace();
1612 1608
		}
......
2019 2015
	 * @return The <code>OriginalDB</code> attribute.
2020 2016
	 * @see MethodMapper
2021 2017
	 */
2022
	@SuppressWarnings("unused")
2018
//	@SuppressWarnings("unused")
2023 2019
	private static String getOriginalDB(IdentifiableEntity<?> identifiableEntity) {
2024 2020
		EnumSet<PesiSource> sources  = getSources(identifiableEntity);
2025 2021
		return PesiTransformer.getOriginalDbBySources(sources);
2026 2022
	}
2027 2023

  
2028
	/**
2029
	 * Returns the <code>LastAction</code> attribute.
2030
	 * @param taxonName The {@link TaxonNameBase TaxonName}.
2031
	 * @return The <code>LastAction</code> attribute.
2032
	 * @see MethodMapper
2033
	 */
2034
	//TODO still in use?
2035
	private static String getLastAction(IdentifiableEntity<?> identEntity) {
2036
		String result = null;
2037
		try {
2038
    		Set<Extension> extensions = identEntity.getExtensions();
2039
    		for (Extension extension : extensions) {
2040
    			if (extension.getType().equals(lastActionExtensionType)) {
2041
    				result = extension.getValue();
2042
    			}
2043
    		}
2044
		} catch (Exception e) {
2045
			e.printStackTrace();
2046
		}
2047
		return result;
2048
	}
2049

  
2050
	/**
2051
	 * Returns the <code>LastActionDate</code> attribute.
2052
	 * @param taxonName The {@link TaxonNameBase TaxonName}.
2053
	 * @return The <code>LastActionDate</code> attribute.
2054
	 * @see MethodMapper
2055
	 */
2056
	//TODO still in use?
2057
	private static DateTime getLastActionDate(IdentifiableEntity<?> identEntity) {
2058
		DateTime result = null;
2059
		try {
2060
			Set<Extension> extensions = identEntity.getExtensions();
2061
			for (Extension extension : extensions) {
2062
				if (extension.getType().equals(lastActionDateExtensionType)) {
2063
					String dateTime = extension.getValue();
2064
					if (dateTime != null) {
2065
						DateTimeFormatter formatter = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.S");
2066
						result = formatter.parseDateTime(dateTime);
2067
					}
2068
				}
2069
			}
2070
		} catch (Exception e) {
2071
			e.printStackTrace();
2072
		}
2073
		return result;
2074
	}
2075

  
2076 2024
	/**
2077 2025
	 * Returns the <code>ExpertName</code> attribute.
2078 2026
	 * @param taxonName The {@link TaxonNameBase TaxonName}.
......
2080 2028
	 * @see MethodMapper
2081 2029
	 */
2082 2030
	@SuppressWarnings("unused")  //for some reason it is also called by getCacheCitation
2083
	private static String getExpertName(TaxonBase<?> taxonName) {
2084
	    String result = null;
2031
	private static String getExpertName(TaxonBase<?> taxon) {
2085 2032
		try {
2033
		    String result = null;
2086 2034
    		if(expertNameExtensionType!=null){  //some databases do not have this extension type
2087
    		    Set<Extension> extensions = taxonName.getExtensions();
2035
    		    Set<Extension> extensions = taxon.getExtensions();
2088 2036
    		    for (Extension extension : extensions) {
2089 2037
    		        if (extension.getType().equals(expertNameExtensionType)) {
2090 2038
    		            result = extension.getValue();
2091 2039
    		        }
2092 2040
    		    }
2093 2041
    		}
2042
    		if (getPesiSources(taxon).contains(PesiSource.EM)){
2043
                return taxon.getSec().getTitleCache();
2044
            }
2045
            return null;
2094 2046
		} catch (Exception e) {
2095 2047
			e.printStackTrace();
2048
			return null;
2096 2049
		}
2097
		return result;
2098 2050
	}
2099 2051

  
2100
	/**
2101
	 * Returns the <code>ExpertFk</code> attribute.
2102
	 * @param taxonName The {@link TaxonNameBase TaxonName}.
2103
	 * @param state The {@link PesiExportState PesiExportState}.
2104
	 * @return The <code>ExpertFk</code> attribute.
2105
	 * @see MethodMapper
2106
	 */
2052
	//TODO change to ExpertGUID
2107 2053
	private static Integer getExpertFk(Reference reference, PesiExportState state) {
2108 2054
		Integer result = state.getDbId(reference);
2109 2055
		return result;
......
2115 2061
	 * @return The <code>SpeciesExpertName</code> attribute.
2116 2062
	 * @see MethodMapper
2117 2063
	 */
2118
	//TODO still in use?
2119
	private static String getSpeciesExpertName(TaxonBase<?> taxonName) {
2120
		String result = null;
2064
	@SuppressWarnings("unused")
2065
	private static String getSpeciesExpertName(TaxonBase<?> taxon) {
2121 2066
		try {
2122
    		Set<Extension> extensions = taxonName.getExtensions();
2123
    		for (Extension extension : extensions) {
2124
    			if (extension.getType().equals(speciesExpertNameExtensionType)) {
2125
    				result = extension.getValue();
2126
    			}
2067
    		Set<Extension> extensions = taxon.getExtensions();
2068
    		if(speciesExpertNameExtensionType!=null){  //some databases do not have this extension type
2069
                for (Extension extension : extensions) {
2070
        			if (extension.getType().equals(speciesExpertNameExtensionType)) {
2071
        				return extension.getValue();
2072
        			}
2073
        		}
2127 2074
    		}
2075
    		if (getPesiSources(taxon).contains(PesiSource.EM)){
2076
    		    return taxon.getSec().getTitleCache();
2077
            }
2078
    		return null;
2128 2079
		} catch (Exception e) {
2129 2080
			e.printStackTrace();
2081
			return null;
2130 2082
		}
2131
		return result;
2132 2083
	}
2133 2084

  
2134 2085
	/**
......
2138 2089
	 * @return The <code>SpeciesExpertFk</code> attribute.
2139 2090
	 * @see MethodMapper
2140 2091
	 */
2092
	//TODO should be changed to SpeciesExpertGUID
2141 2093
	private static Integer getSpeciesExpertFk(Reference reference, PesiExportState state) {
2142 2094
		Integer result = state.getDbId(reference);
2143 2095
		return result;
......
2219 2171
		mapping.addMapper(DbLastActionMapper.NewInstance("LastAction", true));
2220 2172

  
2221 2173
		//experts
2222
		ExtensionType extensionTypeSpeciesExpertName = (ExtensionType)getTermService().find(PesiTransformer.uuidExtSpeciesExpertName);
2223
		mapping.addMapper(DbExtensionMapper.NewInstance(extensionTypeSpeciesExpertName, "SpeciesExpertName"));
2224
		ExtensionType extensionTypeExpertName = (ExtensionType)getTermService().find(PesiTransformer.uuidExtExpertName);
2225
		mapping.addMapper(DbExtensionMapper.NewInstance(extensionTypeExpertName, "ExpertName"));
2174
//		mapping.addMapper(DbExtensionMapper.NewInstance(extensionTypeSpeciesExpertName, "SpeciesExpertName"));
2175
		mapping.addMapper(MethodMapper.NewInstance("SpeciesExpertName", this, TaxonBase.class));
2176
//		ExtensionType extensionTypeExpertName = (ExtensionType)getTermService().find(PesiTransformer.uuidExtExpertName);
2177
//		mapping.addMapper(DbExtensionMapper.NewInstance(extensionTypeExpertName, "ExpertName"));
2178
		mapping.addMapper(MethodMapper.NewInstance("ExpertName", this, TaxonBase.class));
2226 2179

  
2227 2180
		//ParentTaxonFk handled in Phase02 now
2228 2181
		mapping.addMapper(ObjectChangeMapper.NewInstance(TaxonBase.class, TaxonName.class, "Name"));
......
2259 2212
		mapping.addMapper(DbLastActionMapper.NewInstance("LastAction", true));
2260 2213

  
2261 2214
		addNameMappers(mapping);
2262
		//TODO add author mapper
2263

  
2264 2215
		return mapping;
2265 2216
	}
2266 2217

  
cdm-pesi/src/main/java/eu/etaxonomy/cdm/io/pesi/out/PesiTransformer.java
48 48
import eu.etaxonomy.cdm.model.taxon.SynonymType;
49 49
import eu.etaxonomy.cdm.model.taxon.Taxon;
50 50
import eu.etaxonomy.cdm.model.taxon.TaxonBase;
51
import eu.etaxonomy.cdm.model.taxon.TaxonNode;
52
import eu.etaxonomy.cdm.model.taxon.TaxonRelationship;
51 53
import eu.etaxonomy.cdm.model.taxon.TaxonRelationshipType;
52 54
import eu.etaxonomy.cdm.strategy.exceptions.UnknownCdmTypeException;
53 55

  
......
103 105
	// References
104 106
	private static int REF_ARTICLE_IN_PERIODICAL = 1;
105 107
	private static int REF_PART_OF_OTHER = 2;
106
	private static int REF_BOOK = 3;
107
	private static int REF_DATABASE = 4;
108
	public static int REF_BOOK = 3;
109
	public static int REF_DATABASE = 4;
108 110
	private static int REF_INFORMAL = 5;
109 111
	private static int REF_NOT_APPLICABLE = 6;
110 112
	private static int REF_WEBSITE = 7;
111
	private static int REF_PUBLISHED = 8;
112
	private static int REF_JOURNAL = 9;
113
	public static int REF_PUBLISHED_CD = 8;
114
	public static int REF_JOURNAL = 9;
113 115
	public static int REF_UNRESOLVED = 10;
114
	private static int REF_PUBLICATION = 11;
116
	public static int REF_PUBLICATION = 11;
115 117
	public static String REF_STR_UNRESOLVED = "unresolved";
116 118

  
117 119
	private static int LANG_UNKNOWN = -99;
......
154 156
	// TaxonStatus
155 157
	public static int T_STATUS_ACCEPTED = 1;
156 158
	public static int T_STATUS_SYNONYM = 2;
157
	private static int T_STATUS_PARTIAL_SYN = 3;
158
	private static int T_STATUS_PRO_PARTE_SYN = 4;
159
	public static int T_STATUS_PARTIAL_SYN = 3;
160
	public static int T_STATUS_PRO_PARTE_SYN = 4;
159 161
	private static int T_STATUS_UNRESOLVED = 5;
160 162
	private static int T_STATUS_ORPHANED = 6;
161 163
	public static int T_STATUS_UNACCEPTED = 7;
162
	private static int T_STATUS_NOT_ACCEPTED = 8;
164
	private static int T_STATUS_NOT_ACCEPTED_VALUELESS = 8;
163 165

  
164 166
	// TypeDesginationStatus //	 -> not a table anymore
165 167
	private static int TYPE_BY_ORIGINAL_DESIGNATION = 1;
......
192 194
	private static int IS_BLOCKING_NAME_FOR = 20;
193 195
	private static int IS_LECTOTYPE_OF = 61;
194 196
	private static int TYPE_NOT_DESIGNATED = 62;
195
	private static int IS_TAXONOMICALLY_INCLUDED_IN = 101;
197
	public static int IS_TAXONOMICALLY_INCLUDED_IN = 101;
196 198
	public static int IS_SYNONYM_OF = 102;
197 199
	private static int IS_MISAPPLIED_NAME_FOR = 103;
198 200
	private static int IS_PRO_PARTE_SYNONYM_OF = 104;
......
1740 1742
		}
1741 1743
		if (taxonBase.isInstanceOf(Taxon.class)){
1742 1744
			Taxon taxon = CdmBase.deproxy(taxonBase, Taxon.class);
1743
			if (taxon.getTaxonNodes().size() == 0){
1744
				return T_STATUS_NOT_ACCEPTED;
1745
			}else{
1746
				return T_STATUS_ACCEPTED;
1745
			Set<TaxonRelationship> rels = taxon.getRelationsFromThisTaxon();
1746
			Set<TaxonNode> nodes = taxon.getTaxonNodes();
1747
			if (!rels.isEmpty() && !nodes.isEmpty()){
1748
			    logger.warn("Taxon has relations and parent. This is not expected in E+M, but maybe possible in ERMS. Check if taxon status is correct.");
1749
			}else if (rels.isEmpty() && nodes.isEmpty()){
1750
                logger.warn("Taxon has neither relations nor parent. This is not expected. Check if taxon status is correct.");
1751
            }
1752
			if (!rels.isEmpty()){
1753
			    //we expect all rels to have same type, maybe not true
1754
			    UUID relTypeUuid = rels.iterator().next().getType().getUuid();
1755
			    if (TaxonRelationshipType.proParteUuids().contains(relTypeUuid)){
1756
	                return T_STATUS_PRO_PARTE_SYN;
1757
	            }else if (TaxonRelationshipType.partialUuids().contains(relTypeUuid)){
1758
	                return T_STATUS_PARTIAL_SYN;
1759
	            }else if (TaxonRelationshipType.misappliedNameUuids().contains(relTypeUuid)){
1760
	                return T_STATUS_SYNONYM;  //no explicit MAN status exists in PESI
1761
	            }
1747 1762
			}
1763
			if (!nodes.isEmpty()){
1764
			    TaxonNode parentNode = nodes.iterator().next().getParent();
1765
			    if (parentNode.getTaxon() != null && !parentNode.getTaxon().isPublish()){
1766
			        if (parentNode.getTaxon().getUuid().equals(uuidTaxonValuelessEuroMed) ){
1767
			            return T_STATUS_NOT_ACCEPTED_VALUELESS;
1768
			        }
1769
			    }else{
1770
			        return T_STATUS_ACCEPTED;
1771
			    }
1772
	        }
1773
			logger.error("Taxon status could not be defined. This should not happen: " + taxonBase.getTitleCache() );
1774
			return T_STATUS_UNRESOLVED;
1748 1775
		}else if (taxonBase.isInstanceOf(Synonym.class)){
1749
			return T_STATUS_SYNONYM;
1776
			Synonym synonym = CdmBase.deproxy(taxonBase, Synonym.class);
1777
			if (taxonBase2statusFk(synonym.getAcceptedTaxon())== T_STATUS_NOT_ACCEPTED_VALUELESS ){
1778
			    return T_STATUS_NOT_ACCEPTED_VALUELESS;
1779
			}else{
1780
			    return T_STATUS_SYNONYM;
1781
			}
1750 1782
		}else{
1751
			logger.warn("Unknown ");
1783
			logger.warn("Unresolved taxon status.");
1752 1784
			return T_STATUS_UNRESOLVED;
1753 1785
		}
1754 1786
		//TODO
1755
//		public static int T_STATUS_PARTIAL_SYN = 3;
1756
//		public static int T_STATUS_PRO_PARTE_SYN = 4;
1757 1787
//		public static int T_STATUS_UNRESOLVED = 5;
1758 1788
//		public static int T_STATUS_ORPHANED = 6;
1759 1789
	}
......
1810 1840
		} else if (reference.getType().equals(ReferenceType.Journal)) {
1811 1841
			return REF_JOURNAL;
1812 1842
		} else if (reference.getType().equals(ReferenceType.PrintSeries)) {
1813
			return REF_PUBLISHED;
1843
			return REF_PUBLICATION;  //?
1814 1844
		} else if (reference.getType().equals(ReferenceType.Proceedings)) {
1815
			return REF_PUBLISHED;
1845
			return REF_PUBLICATION;  //?
1816 1846
		} else if (reference.getType().equals(ReferenceType.Patent)) {
1817 1847
			return REF_NOT_APPLICABLE;
1818 1848
		} else if (reference.getType().equals(ReferenceType.PersonalCommunication)) {

Also available in: Unified diff