Project

General

Profile

« Previous | Next » 

Revision ba5dcea5

Added by Andreas Müller almost 6 years ago

ref #7334 adapt DwCA and CDM light export to new pro parte synonym handling, also add misapplied name handling to CDM light

View differences:

cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/cdmLight/CdmLightClassificationExport.java
76 76
import eu.etaxonomy.cdm.model.taxon.Synonym;
77 77
import eu.etaxonomy.cdm.model.taxon.Taxon;
78 78
import eu.etaxonomy.cdm.model.taxon.TaxonNode;
79
import eu.etaxonomy.cdm.model.taxon.TaxonRelationship;
80
import eu.etaxonomy.cdm.model.taxon.TaxonRelationshipType;
79 81
import eu.etaxonomy.cdm.strategy.exceptions.UnknownCdmTypeException;
80 82

  
81 83
/**
......
145 147

  
146 148
            @SuppressWarnings("unchecked")
147 149
            TaxonNodeOutStreamPartitioner<XmlExportState> partitioner
148
              = TaxonNodeOutStreamPartitioner.NewInstance(
149
                    this, state, state.getConfig().getTaxonNodeFilter(),
150
                    100, monitor, null);
150
                  = TaxonNodeOutStreamPartitioner.NewInstance(
151
                          this, state, state.getConfig().getTaxonNodeFilter(),
152
                          100, monitor, null);
151 153

  
152 154

  
153 155
                monitor.subTask("Start partitioning");
154 156

  
155 157
                TaxonNode node = partitioner.next();
156 158
                while (node != null){
157
                  handleTaxonNode(state, node);
158
                  node = partitioner.next();
159
                    handleTaxonNode(state, node);
160
                    node = partitioner.next();
159 161
                }
160 162

  
161 163

  
......
175 177
     * @param classificationUuid
176 178
     */
177 179
    private void handleTaxonNode(CdmLightExportState state, TaxonNode taxonNode) {
178
        try {
179
//            TaxonNode taxonNode = getTaxonNodeService().find(taxonNodeUuid);
180 180

  
181 181
            if (taxonNode == null){
182 182
                String message = "TaxonNode for given taxon node UUID not found. ";
183 183
                //TODO
184 184
                state.getResult().addWarning(message);
185 185
            }else{
186
                TaxonNode root = taxonNode;
187
                if (root.hasTaxon()){
188
                    handleTaxon(state, root);
189
                }else{
190
//                    for (TaxonNode child : root.getChildNodes()){
191
//                        handleTaxon(state, child);
192
//                        //TODO progress monitor
193
//                    }
186
                try {
187
                    TaxonNode root = taxonNode;
188
                    if (root.hasTaxon()){
189
                        handleTaxon(state, root);
190
                    }else{
191
    //                    for (TaxonNode child : root.getChildNodes()){
192
    //                        handleTaxon(state, child);
193
    //                        //TODO progress monitor
194
    //                    }
195
                    }
196
                } catch (Exception e) {
197
                    state.getResult().addException(e, "An unexpected error occurred when handling classification " +
198
                            taxonNode.getUuid() + ": " + e.getMessage() + e.getStackTrace());
194 199
                }
195 200
            }
196
        } catch (Exception e) {
197
            state.getResult().addException(e, "An unexpected error occurred when handling classification " +
198
                    taxonNode.getUuid() + ": " + e.getMessage() + e.getStackTrace());
199
        }
200 201
    }
201 202

  
202 203
    /**
......
223 224
                for (Synonym syn : taxon.getSynonyms()){
224 225
                    handleSynonym(state, syn);
225 226
                }
226

  
227
                for (TaxonRelationship rel : taxon.getProParteAndPartialSynonymRelations()){
228
                    handleProPartePartialMisapplied(state, rel);
229
                }
230
                for (TaxonRelationship rel : taxon.getMisappliedNameRelations()){
231
                    handleProPartePartialMisapplied(state, rel);
232
                }
227 233

  
228 234
                CdmLightExportTable table = CdmLightExportTable.TAXON;
229 235
                String[] csvLine = new String[table.getSize()];
......
256 262
        }
257 263
    }
258 264

  
265

  
259 266
    /**
260 267
     * @param state
261 268
     * @param taxon
......
704 711
           csvLine[table.getIndex(CdmLightExportTable.NAME_FK)] = getId(state, name);
705 712
           csvLine[table.getIndex(CdmLightExportTable.SEC_REFERENCE_FK)] = getId(state, synonym.getSec());
706 713
           csvLine[table.getIndex(CdmLightExportTable.SEC_REFERENCE)] = getTitleCache(synonym.getSec());
707
           if (synonym.isProParte()) {
708
        	   csvLine[table.getIndex(CdmLightExportTable.IS_PRO_PARTE)] = "1";
709
           }else {
710
        	   csvLine[table.getIndex(CdmLightExportTable.IS_PRO_PARTE)] = "0";
711
           }
712 714

  
713 715
           state.getProcessor().put(table, synonym, csvLine);
714 716
        } catch (Exception e) {
......
718 720
    }
719 721

  
720 722

  
723
    /**
724
     * Handles Misapplied names (including pro parte and partial as well as
725
     * pro parte and partial synonyms
726
     * @param state
727
     * @param rel
728
     */
729
    private void handleProPartePartialMisapplied(CdmLightExportState state, TaxonRelationship rel) {
730
        try {
731
            Taxon ppSyonym = rel.getFromTaxon();
732
            if (isUnpublished(state.getConfig(), ppSyonym)){
733
                return;
734
            }
735
            TaxonName name = ppSyonym.getName();
736
            handleName(state, name);
737

  
738
            CdmLightExportTable table = CdmLightExportTable.SYNONYM;
739
            String[] csvLine = new String[table.getSize()];
740

  
741
            csvLine[table.getIndex(CdmLightExportTable.SYNONYM_ID)] = getId(state, rel);
742
            csvLine[table.getIndex(CdmLightExportTable.TAXON_FK)] = getId(state, rel.getToTaxon());
743
            csvLine[table.getIndex(CdmLightExportTable.NAME_FK)] = getId(state, name);
744

  
745
            //TODO pro parte synonyms have to references, the synonym relationship reference
746
            //and the sec reference of the Taxon representing the synonym.
747
            //As we currently do have only 1 reference column in CDM light the synonym relationship
748
            //reference is used here. This is according to how pro parte synonyms were mapped to
749
            //concept relationships in #7334
750
            Reference secRef = rel.getCitation();
751
            csvLine[table.getIndex(CdmLightExportTable.SEC_REFERENCE_FK)] = getId(state, secRef);
752
            csvLine[table.getIndex(CdmLightExportTable.SEC_REFERENCE)] = getTitleCache(secRef);
753

  
754
//            Reference secRef = ppSyonym.getSec();
755
//            csvLine[table.getIndex(CdmLightExportTable.SEC_REFERENCE_FK)] = getId(state, secRef);
756
//            csvLine[table.getIndex(CdmLightExportTable.SEC_REFERENCE)] = getTitleCache(secRef);
757
//            Reference synSecRef = rel.getCitation();
758
//            csvLine[table.getIndex(CdmLightExportTable.SYN_SEC_REFERENCE_FK)] = getId(state, secRef);
759
//            csvLine[table.getIndex(CdmLightExportTable.SYN_SEC_REFERENCE)] = getTitleCache(secRef);
760

  
761
            //pro parte type
762
            TaxonRelationshipType type = rel.getType();
763
            csvLine[table.getIndex(CdmLightExportTable.IS_PRO_PARTE)] = type.isProParte()? "1":"0";
764
            csvLine[table.getIndex(CdmLightExportTable.IS_PARTIAL)] = type.isPartial()? "1":"0";
765
            csvLine[table.getIndex(CdmLightExportTable.IS_MISAPPLIED)] = type.isAnyMisappliedName()? "1":"0";
766
            if (type.isPartial()) {
767
                String message = "Partial synonyms/misapplied names not yet handled by CDM light. Created "
768
                    + "pro parte synonym/misapplied name instead for " +  rel.getId();
769
                state.getResult().addWarning(message, "handleProParteSynonym", ppSyonym.getTitleCache());
770
                csvLine[table.getIndex(CdmLightExportTable.IS_PRO_PARTE)] = "1";
771
            }
772

  
773
            state.getProcessor().put(table, ppSyonym, csvLine);
774
         } catch (Exception e) {
775
             state.getResult().addException(e, "An unexpected error occurred when handling "
776
                     + "pro parte/partial synonym relationship " +
777
                     cdmBaseStr(rel) + ": " + e.getMessage());
778
         }
779

  
780
    }
781

  
782

  
721 783
    /**
722 784
     * @param state
723 785
     * @param name
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/cdmLight/CdmLightExportTable.java
46 46
    protected static final String SEC_REFERENCE_FK = "SecReference_FK";
47 47
    protected static final String SEC_REFERENCE = "SecReference";
48 48
    protected static final String IS_PRO_PARTE = "IsProParteSynonym";
49
    protected static final String IS_PARTIAL = "IsPartial";
50
    protected static final String IS_MISAPPLIED = "IsMisapplied";
49 51
    //Reference
50 52
    protected static final String REFERENCE_ID = "Reference_ID";
51 53
    protected static final String BIBLIO_SHORT_CITATION = "BibliographicShortCitation";
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/common/CdmExportBase.java
184 184
                || config.getTaxonNodeFilter().isIncludeUnpublished());
185 185
    }
186 186

  
187

  
188
    /**
189
     * <code>true</code> if neither pro parte synonym or misapplied name has state publish nor
190
     * taxon node filter includes unpublished taxa.
191
     */
192
    protected boolean isUnpublished(CONFIG config, Taxon relatedSynonymOrMisappliedName) {
193
        return ! (relatedSynonymOrMisappliedName.isPublish()
194
                || config.getTaxonNodeFilter().isIncludeUnpublished());
195
    }
196

  
187 197
}
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/dwca/out/DwcaTaxonExport.java
80 80
            TaxonName basionym = name.getBasionym();
81 81
            Classification classification = node.getClassification();
82 82
            if (! state.recordExists(file, taxon)){
83
            	handleTaxonBase(state, record, taxon, name, taxon, parent, basionym, classification, null, false, false);
83
            	handleTaxonBase(state, record, taxon, name, taxon, parent, basionym, classification, null);
84 84
            	PrintWriter writer = createPrintWriter(state, file);
85 85
                record.write(state, writer);
86 86
            	state.addExistingRecord(file, taxon);
......
90 90
            //synonyms
91 91
            if (state.getConfig().isDoSynonyms()){
92 92
                handleSynonyms(state, taxon, file, classification, metaRecord);
93
                //pro parte syonyms
94
                handleProparteSynonyms(state, taxon, file, classification, metaRecord);
93 95
            }
94 96

  
95 97
            //misapplied names
......
97 99
                handleMisapplication(state, taxon, file, classification, metaRecord);
98 100
            }
99 101

  
102

  
103

  
100 104
        } catch (Exception e) {
101 105
            String message = "Unexpected exception: " + e.getMessage();
102 106
            state.getResult().addException(e, message);
......
117 121

  
118 122
		    DwcaTaxonRecord record = new DwcaTaxonRecord(metaRecord, state.getConfig());
119 123
			SynonymType type = synonym.getType();
120
			boolean isProParte = synonym.isProParte();
121
			boolean isPartial = synonym.isPartial();
122 124
			if (type == null){ // should not happen
123 125
				type = SynonymType.SYNONYM_OF();
124 126
			}
......
128 130
			TaxonName basionym = name.getBasionym();
129 131

  
130 132
			if (! state.recordExists(file, synonym)){
131
				handleTaxonBase(state, record, synonym, name, taxon, parent, basionym, classification, type, isProParte, isPartial);
133
				handleTaxonBase(state, record, synonym, name, taxon, parent, basionym, classification, type);
132 134
				PrintWriter writer = createPrintWriter(state, file);
133 135
				record.write(state, writer);
134 136
				state.addExistingRecord(file, synonym);
......
150 152

  
151 153
			if (! state.recordExists(file, misappliedName)){
152 154
				handleTaxonBase(state, record, misappliedName, name, taxon, parent, basionym, classification,
153
				        misappliedNameRel.getType(), false, false);
155
				        misappliedNameRel.getType());
154 156
				PrintWriter writer = createPrintWriter(state, file);
155 157
                record.write(state, writer);
156 158
				state.addExistingRecord(file, misappliedName);
......
158 160
		}
159 161
	}
160 162

  
163
    private void handleProparteSynonyms(DwcaTaxExportState state, Taxon taxon,
164
            DwcaTaxExportFile file, Classification classification, DwcaMetaDataRecord metaRecord) throws FileNotFoundException, UnsupportedEncodingException, IOException {
165

  
166
        Set<TaxonRelationship> proParteRels = taxon.getProParteAndPartialSynonymRelations();
167
        for (TaxonRelationship proParteRel : proParteRels ){
168
            DwcaTaxonRecord record = new DwcaTaxonRecord(metaRecord, state.getConfig());
169
            Taxon proParteSynonym = proParteRel.getFromTaxon();
170
            TaxonName name = proParteSynonym.getName();
171
            //????
172
            Taxon parent = null;
173
            TaxonName basionym = name.getBasionym();
174

  
175
            if (! state.recordExists(file, proParteSynonym)){
176
                handleTaxonBase(state, record, proParteSynonym, name, taxon, parent, basionym, classification,
177
                        proParteRel.getType());
178
                PrintWriter writer = createPrintWriter(state, file);
179
                record.write(state, writer);
180
                state.addExistingRecord(file, proParteSynonym);
181
            }
182
        }
183
    }
184

  
185

  
186

  
161 187
	/**
162 188
	 * @param state
163 189
	 * @param record
......
173 199
	 */
174 200
	private void handleTaxonBase(DwcaTaxExportState state, DwcaTaxonRecord record, TaxonBase<?> taxonBase, TaxonName name,
175 201
			Taxon acceptedTaxon, Taxon parent, TaxonName basionym, Classification classification,
176
			RelationshipTermBase<?> relType, boolean isProParte, boolean isPartial) {
202
			RelationshipTermBase<?> relType) {
177 203
		record.setId(taxonBase.getId());
178 204
		record.setUuid(taxonBase.getUuid());
179 205

  
......
256 282

  
257 283
		record.setNomenclaturalCode(name.getNameType());
258 284
		// ??? TODO Misapplied Names, inferred synonyms
259
		handleTaxonomicStatus(record, name, relType, isProParte, isPartial);
285
		handleTaxonomicStatus(record, name, relType);
260 286
		handleNomStatus(record, taxonBase, name);
261 287

  
262 288
		// TODO we need to differentiate technical
......
368 394
	 * @param isProParte
369 395
	 */
370 396
	private void handleTaxonomicStatus(DwcaTaxonRecord record,
371
			INonViralName name, RelationshipTermBase<?> type,
372
			boolean isProParte, boolean isPartial) {
397
			INonViralName name, RelationshipTermBase<?> type) {
373 398
		if (type == null){
374 399
			record.setTaxonomicStatus(name.getNomenclaturalCode().acceptedTaxonStatusLabel());
375 400
		}else{
......
378 403
				status = "heterotypicSynonym";
379 404
			}else if(type.equals(SynonymType.HOMOTYPIC_SYNONYM_OF())){
380 405
				status = "homotypicSynonym";
406
			}else if(type.equals(TaxonRelationshipType.PRO_PARTE_SYNONYM_FOR())){
407
			    status = "proParteSynonym";
408
			}else if(type.equals(TaxonRelationshipType.PARTIAL_SYNONYM_FOR())){
409
			    String message = "Partial synonym is not part of the gbif toxonomic status vocabulary";
410
			    logger.warn(message);
411
			    status = "proParteMisapplied";
381 412
			}else if(type.equals(TaxonRelationshipType.MISAPPLIED_NAME_FOR())){
382 413
				status = "misapplied";
383 414
			}else if(type.equals(TaxonRelationshipType.PRO_PARTE_MISAPPLIED_NAME_FOR())){
384 415
                status = "proParteMisapplied";
416
            }else if(type.equals(TaxonRelationshipType.PARTIAL_MISAPPLIED_NAME_FOR())){
417
                String message = "Partial misapplied names are not part of the gbif toxonomic status vocabulary";
418
                logger.warn(message);
419
                status = "partialMisapplied";
385 420
            }
386
			if (isProParte){
387
				status = "proParteSynonym";
388
			}else if (isPartial){
389
				String message = "Partial synonym is not part of the gbif toxonomic status vocabulary";
390
				logger.warn(message);
391
				status = "partialSynonym";
392
			}
393 421

  
394 422
			record.setTaxonomicStatus(status);
395 423
		}
cdmlib-io/src/test/java/eu/etaxonomy/cdm/io/cdmLight/out/CdmLightExportTest.java
10 10

  
11 11
import java.io.BufferedReader;
12 12
import java.io.ByteArrayInputStream;
13
import java.io.File;
14 13
import java.io.FileNotFoundException;
15 14
import java.io.IOException;
16 15
import java.io.InputStreamReader;
......
160 159
        @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class, value="/eu/etaxonomy/cdm/database/BlankDataSet.xml")
161 160
        public void testFullTreeWithUnpublished(){
162 161

  
163

  
164 162
            CdmLightExportConfigurator config = new CdmLightExportConfigurator(null);
165 163
            config.setTarget(TARGET.EXPORT_DATA);
166 164
            config.getTaxonNodeFilter().setIncludeUnpublished(true);
......
194 192
                }
195 193
                Assert.assertTrue("There should be 1 synonym", count == 2);
196 194
            } catch (IOException e) {
197
                // TODO Auto-generated catch block
198 195
                e.printStackTrace();
196
                Assert.fail("IO Exception thrown during test.");
199 197
            }
200 198
            byte[] taxon = data.get(CdmLightExportTable.TAXON.getTableName());
201 199
            Assert.assertNotNull("Taxon table must not be null", taxon);
......
243 241
        @DataSet(loadStrategy=CleanSweepInsertLoadStrategy.class, value="/eu/etaxonomy/cdm/database/BlankDataSet.xml")
244 242
        public void testFullData(){
245 243

  
246
            File destinationFolder = null;
247 244
            CdmLightExportConfigurator config = new CdmLightExportConfigurator(null);
248 245
            config.setTarget(TARGET.EXPORT_DATA);
249 246

  
......
270 267
                }
271 268
                Assert.assertTrue("There should be 4 references", count == 5);
272 269
                try{
273
                stream = new ByteArrayInputStream(data.get(CdmLightExportTable.SYNONYM.getTableName()));
274
                Assert.fail("There should not be a synonym table, because the only synonym is not public.");
270
                    stream = new ByteArrayInputStream(data.get(CdmLightExportTable.SYNONYM.getTableName()));
271
                    Assert.fail("There should not be a synonym table, because the only synonym is not public.");
275 272
                }catch(NullPointerException e){
276

  
273
                    //OK, should be thrown
277 274
                }
278 275

  
279 276
            } catch (IOException e) {
280
                // TODO Auto-generated catch block
281
                e.printStackTrace();
277
                throw new RuntimeException(e);
282 278
            }
283

  
284

  
285 279
        }
286 280

  
287 281
        /**
......
301 295
            TaxonName familyName = parser.parseReferencedName("Family L., Sp. Pl. 3: 22. 1752",
302 296
                    NomenclaturalCode.ICNAFP, Rank.FAMILY());
303 297
            setUuid(familyName,"e983cc5e-4c77-4c80-8cb0-73d43df31ef7");
304
            setUuid((Reference)familyName.getNomenclaturalReference(), "b0dd7f4a-0c7f-4372-bc5d-3b676363bc63");
298
            setUuid(familyName.getNomenclaturalReference(), "b0dd7f4a-0c7f-4372-bc5d-3b676363bc63");
305 299
            Taxon family = Taxon.NewInstance(familyName, sec1);
306 300
            setUuid(family,"3162e136-f2e2-4f9a-9010-3f35908fbae1");
307 301
            TaxonNode node1 = classification.addChildTaxon(family, sec1, "22");
......
311 305
            TaxonName genusName = parser.parseReferencedName("Genus Humb., The book of botany 3: 22. 1804",
312 306
                    NomenclaturalCode.ICNAFP, Rank.GENUS());
313 307
            setUuid(genusName,"5e83cc5e-4c77-4d80-8cb0-73d63df35ee3");
314
            setUuid((Reference)genusName.getNomenclaturalReference(), "5ed27f4a-6c7f-4372-bc5d-3b67636abc52");
308
            setUuid(genusName.getNomenclaturalReference(), "5ed27f4a-6c7f-4372-bc5d-3b67636abc52");
315 309
            Taxon genus = Taxon.NewInstance(genusName, sec1);
316 310
            setUuid(genus,"3f52e136-f2e1-4f9a-9010-2f35908fbd39");
317 311

  
......
323 317
            TaxonName speciesName = parser.parseReferencedName("Genus species Mill., The book of botany 3: 22. 1804",
324 318
                    NomenclaturalCode.ICNAFP, Rank.SPECIES());
325 319
            setUuid(speciesName,"f983cc5e-4c77-4c80-8cb0-73d43df31ee9");
326
            setUuid((Reference)speciesName.getNomenclaturalReference(), "a0dd7f4a-0c7f-4372-bc5d-3b676363bc0e");
320
            setUuid(speciesName.getNomenclaturalReference(), "a0dd7f4a-0c7f-4372-bc5d-3b676363bc0e");
327 321
            Taxon species = Taxon.NewInstance(speciesName, sec1);
328 322
            setUuid(species,"9182e136-f2e2-4f9a-9010-3f35908fb5e0");
329 323
            TaxonName synonymName = parser.parseReferencedName("Genus synonym Mill., The book of botany 3: 22. 1804", NomenclaturalCode.ICNAFP, Rank.SPECIES());
......
340 334
            TaxonName subspeciesName = parser.parseReferencedName("Genus species subsp. subspec Mill., The book of botany 3: 22. 1804",
341 335
                    NomenclaturalCode.ICNAFP, Rank.SUBSPECIES());
342 336
            setUuid(subspeciesName,"3483cc5e-4c77-4c80-8cb0-73d43df31ee3");
343
            setUuid((Reference)subspeciesName.getNomenclaturalReference(), "b8dd7f4a-0c7f-4372-bc5d-3b676363bc0f");
337
            setUuid(subspeciesName.getNomenclaturalReference(), "b8dd7f4a-0c7f-4372-bc5d-3b676363bc0f");
344 338

  
345 339
            Taxon subspecies = Taxon.NewInstance(subspeciesName, sec1);
346 340
            setUuid(subspecies, "b2c86698-500e-4efb-b9ae-6bb6e701d4bc");

Also available in: Unified diff