Project

General

Profile

« Previous | Next » 

Revision 4d0649fd

Added by Andreas Müller over 5 years ago

ref #1444 further cleanup ERMS import

View differences:

cdm-pesi/src/main/java/eu/etaxonomy/cdm/app/common/TreeCreator.java
1 1
/**
2 2
* Copyright (C) 2007 EDIT
3
* European Distributed Institute of Taxonomy 
3
* European Distributed Institute of Taxonomy
4 4
* http://www.e-taxonomy.eu
5
* 
5
*
6 6
* The contents of this file are subject to the Mozilla Public License Version 1.1
7 7
* See LICENSE.TXT at the top of this package for the full license terms.
8 8
*/
......
22 22
/**
23 23
 * @author a.mueller
24 24
 * @since 03.07.2008
25
 * @version 1.0
26 25
 */
27 26
public class TreeCreator {
28 27
	@SuppressWarnings("unused")
......
31 30
	public static FeatureTree flatTree(UUID featureTreeUuid, Map<Integer, Feature> featureMap, Object[] featureKeyList){
32 31
		FeatureTree result = FeatureTree.NewInstance(featureTreeUuid);
33 32
		FeatureNode root = result.getRoot();
34
		
33

  
35 34
		for (Object featureKey : featureKeyList){
36 35
			Feature feature = featureMap.get(featureKey);
37 36
			if (feature != null){
38 37
				FeatureNode child = FeatureNode.NewInstance(feature);
39
				root.addChild(child);	
38
				root.addChild(child);
40 39
			}
41 40
		}
42 41
		return result;
43 42
	}
44
	
45
	
43

  
44

  
46 45
	/**
47 46
	 * @param args
48 47
	 */
49 48
	public static void main(String[] args) {
50
		Map<Integer, Feature>  map = new HashMap<Integer, Feature>(null);
49
		Map<Integer, Feature>  map = new HashMap<>(null);
51 50
		map.put(1, Feature.DISTRIBUTION());
52 51
		map.put(2, Feature.ECOLOGY());
53 52

  
54
		Object[] strFeatureList = new Integer[]{1,2}; 
53
		Object[] strFeatureList = new Integer[]{1,2};
55 54

  
56 55
		FeatureTree tree = TreeCreator.flatTree(UUID.randomUUID(), map, strFeatureList);
57 56
		System.out.println(tree.getRootChildren());
cdm-pesi/src/main/java/eu/etaxonomy/cdm/io/pesi/erms/ErmsImportBase.java
115 115
	}
116 116

  
117 117
	@Override
118
    public boolean doPartition(ResultSetPartitioner partitioner, ErmsImportState state) {
118
    public boolean doPartition(@SuppressWarnings("rawtypes") ResultSetPartitioner partitioner, ErmsImportState state) {
119 119
		boolean success = true ;
120 120
		Set<CdmBase> objectsToSave = new HashSet<>();
121 121

  
......
125 125
		ResultSet rs = partitioner.getResultSet();
126 126
		try{
127 127
			while (rs.next()){
128
				success &= mapping.invoke(rs,objectsToSave);
128
				success &= mapping.invoke(rs, objectsToSave);
129 129
			}
130 130
		} catch (SQLException e) {
131 131
			logger.error("SQLException:" +  e);
......
137 137
		return success;
138 138
	}
139 139

  
140

  
141

  
142 140
	/**
143 141
	 * @return
144 142
	 */
cdm-pesi/src/main/java/eu/etaxonomy/cdm/io/pesi/erms/ErmsImportConfigurator.java
23 23
import eu.etaxonomy.cdm.model.reference.Reference;
24 24
import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
25 25

  
26

  
27 26
/**
28 27
 * @author a.mueller
29 28
 * @since 20.03.2008
30 29
 */
31 30
public class ErmsImportConfigurator
32 31
        extends DbImportConfiguratorBase<ErmsImportState>{
32

  
33 33
    private static final long serialVersionUID = 5434106058744720246L;
34 34

  
35 35
    @SuppressWarnings("unused")
36 36
	private static Logger logger = Logger.getLogger(ErmsImportConfigurator.class);
37 37

  
38 38
	public static ErmsImportConfigurator NewInstance(Source ermsSource, ICdmDataSource destination){
39
			return new ErmsImportConfigurator(ermsSource, destination);
39
	    return new ErmsImportConfigurator(ermsSource, destination);
40 40
	}
41 41

  
42 42
	/* Max number of records to be saved with one service call */
......
57 57

  
58 58
	private static IInputTransformer defaultTransformer = new ErmsTransformer();
59 59

  
60
	@Override
60
	@SuppressWarnings("unchecked")
61
    @Override
61 62
    protected void makeIoClassList(){
62 63
		ioClassList = new Class[]{
63 64
				ErmsGeneralImportValidator.class
......
78 79
		};
79 80
	}
80 81

  
81

  
82
	/* (non-Javadoc)
83
	 * @see eu.etaxonomy.cdm.io.common.IImportConfigurator#getNewState()
84
	 */
85 82
	@Override
86 83
    public ImportStateBase getNewState() {
87 84
		return new ErmsImportState(this);
88 85
	}
89 86

  
90

  
91

  
92 87
	/**
93 88
	 * @param berlinModelSource
94 89
	 * @param sourceReference
......
98 93
	   super(ermsSource, destination, NomenclaturalCode.ICZN, defaultTransformer);//default for ERMS
99 94
	}
100 95

  
101

  
102 96
	@Override
103 97
    public Source getSource() {
104 98
		return super.getSource();
......
108 102
		super.setSource(berlinModelSource);
109 103
	}
110 104

  
111
	/* (non-Javadoc)
112
	 * @see eu.etaxonomy.cdm.io.tcsrdf.IImportConfigurator#getSourceReference()
113
	 */
114 105
	@Override
115 106
    public Reference getSourceReference() {
116 107
		if (sourceReference == null){
......
125 116
		return sourceReference;
126 117
	}
127 118

  
128

  
129
	/* (non-Javadoc)
130
	 * @see eu.etaxonomy.cdm.io.common.IImportConfigurator#getSourceNameString()
131
	 */
132 119
	@Override
133 120
    public String getSourceNameString() {
134 121
		if (this.getSource() == null){
......
138 125
		}
139 126
	}
140 127

  
141
	/**
142
	 * @return the userTransformationMethod
143
	 */
144 128
	@Override
145 129
    public Method getUserTransformationMethod() {
146 130
		return userTransformationMethod;
147 131
	}
148

  
149
	/**
150
	 * @param userTransformationMethod the userTransformationMethod to set
151
	 */
152 132
	@Override
153 133
    public void setUserTransformationMethod(Method userTransformationMethod) {
154 134
		this.userTransformationMethod = userTransformationMethod;
155 135
	}
156 136

  
157

  
158
	/**
159
	 * @return the limitSave
160
	 */
161 137
	@Override
162 138
    public int getRecordsPerTransaction() {
163 139
		return recordsPerTransaction;
164 140
	}
165

  
166
	/**
167
	 * @param limitSave the limitSave to set
168
	 */
169 141
	@Override
170 142
    public void setRecordsPerTransaction(int recordsPerTransaction) {
171 143
		this.recordsPerTransaction = recordsPerTransaction;
172 144
	}
173 145

  
174
	/**
175
	 * @param doVernaculars the doVernaculars to set
176
	 */
177 146
	public void setDoVernaculars(boolean doVernaculars) {
178 147
		this.doVernaculars = doVernaculars;
179 148
	}
180

  
181
	/**
182
	 * @return the doVernaculars
183
	 */
184 149
	public boolean isDoVernaculars() {
185 150
		return doVernaculars;
186 151
	}
187 152

  
188

  
189

  
190
	/**
191
	 * @param doLinks the doLinks to set
192
	 */
193 153
	public void setDoLinks(boolean doLinks) {
194 154
		this.doLinks = doLinks;
195 155
	}
196

  
197

  
198

  
199
	/**
200
	 * @return the doLinks
201
	 */
202 156
	public boolean isDoLinks() {
203 157
		return doLinks;
204 158
	}
205 159

  
206

  
207

  
208
	/**
209
	 * @param doNotes the doNotes to set
210
	 */
211 160
	public void setDoNotes(boolean doNotes) {
212 161
		this.doNotes = doNotes;
213 162
	}
214

  
215

  
216

  
217
	/**
218
	 * @return the doNotes
219
	 */
220 163
	public boolean isDoNotes() {
221 164
		return doNotes;
222 165
	}
223 166

  
224

  
225

  
226
	/**
227
	 * @param doImages the doImages to set
228
	 */
229 167
	public void setDoImages(boolean doImages) {
230 168
		this.doImages = doImages;
231 169
	}
232

  
233

  
234

  
235
	/**
236
	 * @return the doImages
237
	 */
238 170
	public boolean isDoImages() {
239 171
		return doImages;
240 172
	}
241 173

  
242

  
243 174
	public boolean isDoOccurrence() {
244 175
		return doOccurrence;
245 176
	}
......
247 178
		this.doOccurrence = doOccurrence;
248 179
	}
249 180

  
250

  
251 181
	public DO_REFERENCES getDoReferences() {
252 182
		return doReferences;
253 183
	}
......
269 199
		this.doRelTaxa = doRelTaxa;
270 200
	}
271 201

  
272

  
273

  
274 202
}
cdm-pesi/src/main/java/eu/etaxonomy/cdm/io/pesi/erms/ErmsImportRankMap.java
98 98
	private Map<Integer, Rank> makeKingdomMap(Map<Integer, Map<Integer, Rank>> rankMap, Integer rankId) {
99 99
		Map<Integer, Rank> result = rankMap.get(rankId);
100 100
		if (result == null){
101
			result = new HashMap<Integer, Rank>();
101
			result = new HashMap<>();
102 102
			rankMap.put(rankId, result);
103 103
		}
104 104
		return result;
105 105
	}
106 106

  
107

  
108
	@Override
109
	protected boolean doCheck(ErmsImportState state) {
110
		IOValidator<ErmsImportState> rankImport = new ErmsRankImportValidator();
111
		return rankImport.validate(state);
112
	}
113

  
114

  
115 107
	@Override
116 108
	protected String getRecordQuery(ErmsImportConfigurator config) {
117
		// TODO Auto-generated method stub
118
		return null;
119
	}
120

  
121

  
122
	@Override
123
	protected boolean isIgnore(ErmsImportState state) {
124
		return false;  //should always be called
109
		return null;   // not needed
125 110
	}
126 111

  
127 112
	@Override
......
129 114
		return null;  // not needed
130 115
	}
131 116

  
132
	public Rank createObject(ResultSet rs, ErmsImportState state)
133
			throws SQLException {
134
		return null;  // not needed
135
	}
136

  
137

  
138 117
	@Override
139 118
	protected DbImportMapping<?, ?> getMapping() {
140 119
		return null;  //not needed
141 120
	}
142 121

  
122
    @Override
123
    protected boolean doCheck(ErmsImportState state) {
124
        IOValidator<ErmsImportState> rankImport = new ErmsRankImportValidator();
125
        return rankImport.validate(state);
126
    }
127

  
128
    @Override
129
    protected boolean isIgnore(ErmsImportState state) {
130
        return false;  //should always be called
131
    }
143 132
}
cdm-pesi/src/main/java/eu/etaxonomy/cdm/io/pesi/erms/ErmsTaxonImport.java
45 45
import eu.etaxonomy.cdm.model.taxon.TaxonBase;
46 46
import eu.etaxonomy.cdm.strategy.cache.name.TaxonNameDefaultCacheStrategy;
47 47

  
48

  
49 48
/**
50 49
 * @author a.mueller
51 50
 * @since 20.02.2010
......
80 79
	@Override
81 80
    protected DbImportMapping<ErmsImportState, ErmsImportConfigurator> getMapping() {
82 81
		if (mapping == null){
83
			mapping = new DbImportMapping<ErmsImportState, ErmsImportConfigurator>();
82
			mapping = new DbImportMapping<>();
84 83

  
85 84
			mapping.addMapper(DbImportObjectCreationMapper.NewInstance(this, "id", TAXON_NAMESPACE)); //id + tu_status
86 85
			UUID tsnUuid = ErmsTransformer.uuidTsn;
......
94 93
			mapping.addMapper(DbImportExtensionMapper.NewInstance("tu_displayname", displayNameExtType));
95 94
			ExtensionType fuzzyNameExtType = getExtensionType(ErmsTransformer.uuidFuzzyName, "fuzzy name", "fuzzy name", "fuzzy name");
96 95
		//	mapping.addMapper(DbImportExtensionMapper.NewInstance("tu_fuzzyname", fuzzyNameExtType));
97
			mapping.addMapper(DbImportStringMapper.NewInstance("tu_authority", "(NonViralName)name.authorshipCache"));
96
			mapping.addMapper(DbImportStringMapper.NewInstance("tu_authority", "name.authorshipCache"));
98 97

  
99 98
			ExtensionType fossilStatusExtType = getExtensionType(ErmsTransformer.uuidFossilStatus, "fossil status", "fossil status", "fos. stat.");
100 99
			mapping.addMapper(DbImportExtensionMapper.NewInstance("fossil_name", fossilStatusExtType));
......
111 110
			mapping.addMapper(DbImportMarkerMapper.NewInstance("tu_fresh", ErmsTransformer.uuidMarkerFreshwater, "freshwater", "fresh", "fresh", null));
112 111
			mapping.addMapper(DbImportMarkerMapper.NewInstance("tu_terrestrial", ErmsTransformer.uuidMarkerTerrestrial, "terrestrial", "terrestrial", "terrestrial", null));
113 112

  
114

  
115 113
//			UUID hiddenUuid = ErmsTransformer.uuidHidden;
116 114
//			mapping.addMapper(DbImportMarkerCreationMapper.Mapper.NewInstance("qualitystatus_name", qualityUuid, "quality status", "quality status", "quality status")); //checked by Tax Editor ERMS1.1, Added by db management team (2x), checked by Tax Editor
117 115

  
......
152 150
	}
153 151

  
154 152
	private Set<Integer> getAcceptedTaxaKeys(ErmsImportState state) {
155
		Set<Integer> result = new HashSet<Integer>();
153
		Set<Integer> result = new HashSet<>();
156 154
		String parentCol = "tu_parent";
157 155
		String accCol = " tu_acctaxon";
158 156
		String idCol = " id ";
......
160 158
		String taxonTable = "tu";
161 159
		String vernacularsTable = "vernaculars";
162 160
		String distributionTable = "dr";
163
		String sql = " SELECT DISTINCT %s FROM %s  " +
164
				" UNION  SELECT %s FROM %s WHERE %s is NULL" +
165
				" UNION  SELECT DISTINCT %s FROM %s " +
166
				" UNION  SELECT DISTINCT %s FROM %s " +
167
				" UNION  SELECT DISTINCT %s FROM %s ";
161
		String sql =
162
		        " SELECT DISTINCT %s FROM %s  "  //fk to parent
163
		        + " UNION  SELECT %s FROM %s WHERE %s is NULL" //id of taxa not having accepted taxon
164
		        + " UNION  SELECT DISTINCT %s FROM %s "  //fk to accepted taxon
165
		        + " UNION  SELECT DISTINCT %s FROM %s " //vernaculars
166
		        + " UNION  SELECT DISTINCT %s FROM %s ";  //distributions
168 167
		sql = String.format(sql,
169 168
				parentCol, taxonTable,
170 169
				idCol, taxonTable, accCol,
cdm-pesi/src/main/java/eu/etaxonomy/cdm/io/pesi/erms/ErmsTaxonRelationImport.java
55 55
		super(pluralString, dbTableName, cdmTargetClass);
56 56
	}
57 57

  
58

  
59 58
	@Override
60 59
	protected DbImportMapping<ErmsImportState, ErmsImportConfigurator> getMapping() {
61 60
		if (mapping == null){
......
98 97
		String nameSpace;
99 98
		Class<?> cdmClass;
100 99
		Set<String> idSet;
101
		Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<Object, Map<String, ? extends CdmBase>>();
100
		Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<>();
102 101

  
103 102
		try{
104
			Set<String> taxonIdSet = new HashSet<String>();
105
			Set<String> nameIdSet = new HashSet<String>();
103
			Set<String> taxonIdSet = new HashSet<>();
104
			Set<String> nameIdSet = new HashSet<>();
106 105
			while (rs.next()){
107 106
				handleForeignKey(rs, taxonIdSet, "parentId");
108 107
				handleForeignKey(rs, taxonIdSet, "accParentId");
......
128 127
            Map<String, TaxonBase<?>> taxonMap = (Map<String, TaxonBase<?>>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
129 128
			result.put(nameSpace, taxonMap);
130 129

  
130
			return result;
131

  
131 132
		} catch (SQLException e) {
132 133
			throw new RuntimeException(e);
133 134
		}
134
		return result;
135 135
	}
136 136

  
137 137
	@Override
cdm-pesi/src/main/java/eu/etaxonomy/cdm/io/pesi/erms/validation/ErmsReferenceImportValidator.java
1 1
/**
2 2
* Copyright (C) 2007 EDIT
3
* European Distributed Institute of Taxonomy 
3
* European Distributed Institute of Taxonomy
4 4
* http://www.e-taxonomy.eu
5
* 
5
*
6 6
* The contents of this file are subject to the Mozilla Public License Version 1.1
7 7
* See LICENSE.TXT at the top of this package for the full license terms.
8 8
*/
......
14 14

  
15 15
import org.apache.log4j.Logger;
16 16

  
17

  
18 17
import eu.etaxonomy.cdm.io.common.IOValidator;
19 18
import eu.etaxonomy.cdm.io.common.Source;
20 19
import eu.etaxonomy.cdm.io.pesi.erms.ErmsImportConfigurator;
21 20
import eu.etaxonomy.cdm.io.pesi.erms.ErmsImportState;
22 21

  
23 22
/**
23
 * ERMS reference validator.
24
 * NOT YET USED.
25
 *
24 26
 * @author a.mueller
25 27
 * @since 17.02.2010
26
 * @version 1.0
27 28
 */
28 29
public class ErmsReferenceImportValidator implements IOValidator<ErmsImportState>{
29 30
	private static final Logger logger = Logger.getLogger(ErmsReferenceImportValidator.class);
30 31

  
31
	public boolean validate(ErmsImportState state){
32
	@Override
33
    public boolean validate(ErmsImportState state){
32 34
		boolean result = true;
33 35
		ErmsImportConfigurator config = state.getConfig();
34 36
		logger.info("Checking for references not yet fully implemented");
35 37
//		result &= checkTaxonStatus(config);
36
//		result &= checkInactivated(config);
37 38
		return result;
38 39
	}
39
	
40

  
40 41
	private boolean checkTaxonStatus(ErmsImportConfigurator bmiConfig){
41 42
		try {
42 43
			boolean result = true;
43 44
			Source source = bmiConfig.getSource();
44 45
			String strSQL = " SELECT RelPTaxon.RelQualifierFk, RelPTaxon.relPTaxonId, PTaxon.PTNameFk, PTaxon.PTRefFk, PTaxon_1.PTNameFk AS Expr1, PTaxon.RIdentifier, PTaxon_1.RIdentifier AS Expr3, Name.FullNameCache "  +
45
				" FROM RelPTaxon " + 
46
					" INNER JOIN PTaxon ON RelPTaxon.PTNameFk1 = PTaxon.PTNameFk AND RelPTaxon.PTRefFk1 = PTaxon.PTRefFk " + 
47
					" INNER JOIN PTaxon AS PTaxon_1 ON RelPTaxon.PTNameFk2 = PTaxon_1.PTNameFk AND RelPTaxon.PTRefFk2 = PTaxon_1.PTRefFk  " + 
46
				" FROM RelPTaxon " +
47
					" INNER JOIN PTaxon ON RelPTaxon.PTNameFk1 = PTaxon.PTNameFk AND RelPTaxon.PTRefFk1 = PTaxon.PTRefFk " +
48
					" INNER JOIN PTaxon AS PTaxon_1 ON RelPTaxon.PTNameFk2 = PTaxon_1.PTNameFk AND RelPTaxon.PTRefFk2 = PTaxon_1.PTRefFk  " +
48 49
					" INNER JOIN Name ON PTaxon.PTNameFk = Name.NameId " +
49 50
				" WHERE (dbo.PTaxon.StatusFk = 1) AND ((RelPTaxon.RelQualifierFk = 7) OR (RelPTaxon.RelQualifierFk = 6) OR (RelPTaxon.RelQualifierFk = 2)) ";
50 51
			ResultSet rs = source.getResultSet(strSQL);
......
62 63
				int refFk = rs.getInt("PTRefFk");
63 64
				int relPTaxonId = rs.getInt("relPTaxonId");
64 65
				String taxonName = rs.getString("FullNameCache");
65
				
66
				System.out.println("RIdentifier:" + rIdentifier + "\n  name: " + nameFk + 
66

  
67
				System.out.println("RIdentifier:" + rIdentifier + "\n  name: " + nameFk +
67 68
						"\n  taxonName: " + taxonName + "\n  refId: " + refFk + "\n  RelPTaxonId: " + relPTaxonId );
68 69
				result = firstRow = false;
69 70
			}
70 71
			if (i > 0){
71 72
				System.out.println(" ");
72 73
			}
73
			
74

  
74 75
			return result;
75 76
		} catch (SQLException e) {
76 77
			e.printStackTrace();
......
78 79
		}
79 80
	}
80 81

  
81

  
82

  
83 82
}
cdm-pesi/src/main/java/eu/etaxonomy/cdm/io/pesi/erms/validation/ErmsVernacularImportValidator.java
1 1
/**
2 2
* Copyright (C) 2007 EDIT
3
* European Distributed Institute of Taxonomy 
3
* European Distributed Institute of Taxonomy
4 4
* http://www.e-taxonomy.eu
5
* 
5
*
6 6
* The contents of this file are subject to the Mozilla Public License Version 1.1
7 7
* See LICENSE.TXT at the top of this package for the full license terms.
8 8
*/
......
18 18
/**
19 19
 * @author a.mueller
20 20
 * @since 12.03.2010
21
 * @version 1.0
22 21
 */
23 22
public class ErmsVernacularImportValidator  implements IOValidator<ErmsImportState>{
24 23
	private static final Logger logger = Logger.getLogger(ErmsVernacularImportValidator.class);
25
	
26
	public boolean validate(ErmsImportState state){
24

  
25
	@Override
26
    public boolean validate(ErmsImportState state){
27 27
		boolean result = true;
28 28
		ErmsImportConfigurator config = state.getConfig();
29 29
		logger.info("Checking for vernaculars not yet implemented");

Also available in: Unified diff