Project

General

Profile

Revision f6f52f73

IDf6f52f73cb81a380103edc6f453c9e8acfe5c679
Parent 5462bc2b
Child 0455fc3f

Added by Andreas Müller almost 4 years ago

ref #7216 adapt Cuba import for pteridophytae import

View differences:

app-import/src/main/java/eu/etaxonomy/cdm/app/common/CdmDestinations.java
358 358
		return makeDestination(dbType, cdmServer, cdmDB, -1, cdmUserName, null);
359 359
	}
360 360

  
361
    public static ICdmDataSource cdm_local_cuba(){
362
        DatabaseTypeEnum dbType = DatabaseTypeEnum.MySQL;
363
        String cdmServer = "127.0.0.1";
364
        String cdmDB = "cdm_local_flora_cuba";
365
        String cdmUserName = "edit";
366
        return makeDestination(dbType, cdmServer, cdmDB, -1, cdmUserName, null);
367
    }
368

  
361 369
	public static ICdmDataSource cdm_local_caryo(){
362 370
		DatabaseTypeEnum dbType = DatabaseTypeEnum.MySQL;
363 371
		String cdmServer = "127.0.0.1";
app-import/src/main/java/eu/etaxonomy/cdm/app/cuba/CubaActivator.java
41 41
	//database validation status (create, update, validate ...)
42 42
	static DbSchemaValidation hbm2dll = DbSchemaValidation.VALIDATE;
43 43

  
44
    static final ICdmDataSource cdmDestination = CdmDestinations.localH2();
44
//  static final ICdmDataSource cdmDestination = CdmDestinations.localH2();
45 45
//	static final ICdmDataSource cdmDestination = CdmDestinations.cdm_test_local_mysql_test();
46
//    static final ICdmDataSource cdmDestination = CdmDestinations.cdm_cuba_production();
46
    static final ICdmDataSource cdmDestination = CdmDestinations.cdm_local_cuba();
47
//  static final ICdmDataSource cdmDestination = CdmDestinations.cdm_cuba_production();
47 48

  
48 49
	static boolean invers = true;
49 50

  
......
64 65
    boolean doOrchidaceae = include;
65 66
    boolean doRubiaceae = include;
66 67
    boolean doUrticaceae = include;
68
    boolean doPteridophyta = !include;
67 69

  
68 70
    static boolean include = !invers;
69 71

  
......
81 83
	static final CHECK check = CHECK.IMPORT_WITHOUT_CHECK;
82 84

  
83 85
	boolean doVocabularies = (hbm2dll == DbSchemaValidation.CREATE);
84
	static final boolean doTaxa = false;
85
	static final boolean doDeduplicate = true;
86
	static final boolean doTaxa = true;
87
	static final boolean doDeduplicate = false;
88

  
89
	static final boolean doAltFlorasSanchez2017 = true;
90
	static final boolean doAltFlorasFRC = true;
91
	static final boolean doAltFlorasFC = false;
92
	static final boolean doAltFlorasAS = false;
86 93

  
87 94

  
88 95
	private void doImport(ICdmDataSource cdmDestination){
......
99 106
		config.setDbSchemaValidation(hbm2dll);
100 107
		config.setSourceReferenceTitle(sourceReferenceTitle);
101 108
		config.setDoVocabularies(doVocabularies);
109
		config.setDoAltFlorasAS(doAltFlorasAS);
110
		config.setDoAltFlorasFC(doAltFlorasFC);
111
		config.setDoAltFlorasFRC(doAltFlorasFRC);
112
		config.setDoAltFlorasSanchez2017(doAltFlorasSanchez2017);
102 113

  
103
		CdmDefaultImport<CubaImportConfigurator> myImport = new CdmDefaultImport<CubaImportConfigurator>();
114
		CdmDefaultImport<CubaImportConfigurator> myImport = new CdmDefaultImport<>();
104 115

  
105 116

  
106 117
		//...
......
155 166
        if (doUrticaceae){
156 167
            doSingleSource(urticaceae(), config, myImport, doVocabularies);
157 168
        }
169
        if (doPteridophyta){
170
            doSingleSource(pteridophyta(), config, myImport, doVocabularies);
171
        }
158 172

  
159 173

  
160 174
		//deduplicate
......
303 317
    public static URI rubiaceae() {
304 318
        return URI.create("file:////BGBM-PESIHPC/Cuba/Rubiaceae.xlsx");
305 319
    }
320
    //Rubiaceae
321
    public static URI pteridophyta() {
322
        return URI.create("file:////BGBM-PESIHPC/Cuba/Pteridophyta.xlsx");
323
    }
306 324

  
307 325
	/**
308 326
	 * @param args
app-import/src/main/java/eu/etaxonomy/cdm/io/cuba/CubaExcelImport.java
90 90
    private static INonViralNameParser<?> nameParser = NonViralNameParserImpl.NewInstance();
91 91
    private static NomenclaturalCode nc = NomenclaturalCode.ICNAFP;
92 92

  
93
    private  static List<String> expectedKeys= Arrays.asList(new String[]{
94
            "Fam. default","Fam. FRC","Fam. A&S","Fam. FC",
93
    private  static List<String> expectedKeys = Arrays.asList(new String[]{
94
            "Fam. default","Fam. FRC","Fam. A&S","Fam. FC","Fam. Sánchez 2017",
95 95
            "Taxón","(Notas)","Syn.","End","Ind","Ind? D","Nat","Dud P","Adv","Cult C","CuW","PR PR*","Art","Hab(*)","May","Mat","IJ","CuC","VC","Ci","SS","CA","Cam","LT","CuE","Gr","Ho","SC","Gu","Esp","Ja","PR","Men","Bah","Cay","AmN","AmC","AmS","VM"});
96 96

  
97 97
	@Override
......
407 407
            misappliedNameTaxon.setDoubtful(doubtful);
408 408
            if (secondPart.startsWith("sensu")){
409 409
                secondPart = secondPart.substring(5).trim();
410
                if (secondPart.contains(" ")){
411
                    logger.warn(line + "CHECK: Second part contains more than 1 word. Check if this is correct: " + secondPart);
410
                Reference sensu;
411
                if (isConcreteReference(secondPart)) {
412
                    List<Reference> sensuRefs = getConcreteReferences(secondPart, state);
413
                    if (sensuRefs.size() > 1){
414
                        logger.warn(line + "MAs > 1 not yet handled: " + secondPart);
415
                    }
416
                    sensu = sensuRefs.get(0);
417
                }else{
418
                    if (secondPart.contains(" ")){
419
                        logger.warn(line + "CHECK: Second part contains more than 1 word. Check if this is correct: " + secondPart);
420
                    }
421
                    sensu = ReferenceFactory.newGeneric();
422
                    Team team = Team.NewTitledInstance(secondPart, null);
423
                    sensu.setAuthorship(team);
412 424
                }
413
                Reference sensu = ReferenceFactory.newGeneric();
414
                Team team = Team.NewTitledInstance(secondPart, null);
415
                sensu.setAuthorship(team);
416 425
                misappliedNameTaxon.setSec(sensu);
417 426
            }else if (secondPart.matches(auctRegExStr)){
418 427
                secondPart = secondPart.replace("p. p.", "p.p.");
......
484 493
    }
485 494

  
486 495
    /**
496
     * @param secondPart
497
     * @return
498
     */
499
    private boolean isConcreteReference(String secondPart) {
500
        boolean result = false;
501
        String[] splits = secondPart.split(",");
502
        for (String split : splits){
503
            split = split.trim();
504
            result = split.equals("Griseb. 2") || split.equals("Sauv. 3")
505
                    || split.equals("Grisebach 5") || split.equals("Griseb. 78") ;
506
            if (result == false){
507
                return result;
508
            }
509
        }
510
        return result;
511
    }
512

  
513
    private List<Reference> getConcreteReferences(String secondPart, CubaImportState state) {
514
        List<Reference> result = new ArrayList<>();
515
        String[] splits = secondPart.split(",");
516
        for (String split : splits){
517
            split = split.trim();
518
            if (split.equals("Griseb. 2")){
519
                result.add(getSourceByNumber("2", state));
520
            }else if (split.equals("Sauv. 3")){
521
                result.add(getSourceByNumber("3", state));
522
            }else if (split.equals("Grisebach 5")){
523
                result.add(getSourceByNumber("5", state));
524
            }else if (split.equals("Griseb. 78")){
525
                result.add(getSourceByNumber("78", state));
526
            }else{
527
                logger.warn("Concrete reference does not match: " + split);
528
            }
529
        }
530
        return result;
531
    }
532

  
533

  
534
    /**
487 535
     * @param state
488 536
     * @param list
489 537
     * @param isFirstSynonym
......
565 613
            }
566 614
            TaxonDescription desc = getTaxonDescription(state.getCurrentTaxon(), false, true);
567 615
            PresenceAbsenceTerm status =  makeProvinceStatus(areaKey, record, state);
568
            if (status == null){
616
            Reference source = getAreaSourceByNumber(areaKey, record, state);
617
            if (status == null && source == null){
569 618
                logger.warn(state.getCurrentLine() + ": Province distribution status could not be defined: " + record.get(areaKey));
619
            }else if (status == null){
620
                status = PresenceAbsenceTerm.NATIVE();
570 621
            }
571 622
            Distribution distribution = Distribution.NewInstance(area, status);
572 623
            desc.addElement(distribution);
573 624
            distribution.addSource(makeDescriptionSource(state));
625
            if (source != null){
626
                distribution.addPrimaryTaxonomicSource(source, null);
627
            }
574 628
        } catch (UndefinedTransformerMethodException e) {
575 629
            e.printStackTrace();
576 630
        }
......
579 633

  
580 634

  
581 635
    /**
636
     * @param areaKey
637
     * @param record
638
     * @param state
639
     */
640
    private Reference getAreaSourceByNumber(String areaKey, HashMap<String, String> record, CubaImportState state) {
641
        String statusStr = record.get(areaKey);
642
        if (statusStr == null){
643
            return null;
644
        }else{
645
            statusStr = statusStr.trim();
646
        }
647
        if ("p 78".equals(statusStr)){
648
            statusStr = "78";
649
        }else if ("– 7".equals(statusStr)){
650
            statusStr = "7";
651
        }
652
        return getSourceByNumber(statusStr, state);
653
    }
654

  
655
    private Reference getSourceByNumber(String number, CubaImportState state) {
656
        if ("78".equals(number)){
657
            return makeReference(state, CubaTransformer.uuidRefPteridophyta78);
658
        }else if ("1".equals(number)){
659
            return makeReference(state, CubaTransformer.uuidRefPteridophyta2);
660
        }else if ("2".equals(number)){
661
            return makeReference(state, CubaTransformer.uuidRefPteridophyta2);
662
        }else if ("3".equals(number)){
663
            return makeReference(state, CubaTransformer.uuidRefPteridophyta3);
664
        }else if ("5".equals(number)){
665
            return makeReference(state, CubaTransformer.uuidRefPteridophyta5);
666
        }else if ("8".equals(number)){
667
            return makeReference(state, CubaTransformer.uuidRefPteridophyta8_82);
668
        }else if ("7".equals(number)){
669
            return makeReference(state, CubaTransformer.uuidRefPteridophyta7);
670
        }
671
        return null;
672
    }
673

  
674

  
675
    /**
582 676
     * @param synonymStr
583 677
     * @param state
584 678
     * @param homonyms
......
898 992
                genus = Taxon.NewInstance(name, sec);
899 993
                genus.addSource(makeOriginalSource(state));
900 994
                higherNode = familyNode.addChildTaxon(genus, null, null);
995
                getTaxonNodeService().saveOrUpdate(higherNode);
901 996
                state.putHigherTaxon(genusStr, genus);
902 997
            }
903 998
        }
904 999
        taxon.addSource(makeOriginalSource(state));
905 1000

  
906 1001
        TaxonNode newNode = higherNode.addChildTaxon(taxon, null, null);
1002
        getTaxonNodeService().saveOrUpdate(newNode);
907 1003
        if(isAbsent){
908 1004
            botanicalName.setTitleCache(taxonStrOrig, true);
909 1005
            newNode.setExcluded(true);
......
1083 1179
            family = Taxon.NewInstance(name, sec);
1084 1180
            ITaxonTreeNode rootNode = getClassification(state);
1085 1181
            familyNode = rootNode.addChildTaxon(family, sec, null);
1182
            this.getTaxonNodeService().saveOrUpdate(familyNode);
1086 1183
            state.putHigherTaxon(familyStr, family);
1087 1184

  
1088 1185
        }
......
1154 1251
                            hasExternalPresence = true;
1155 1252
                            if (state.isEndemic()){
1156 1253
                                String name = taxon.getName().getTitleCache();
1157
                                logger.error(state.getCurrentLine() +": Taxon ("+name+")is endemic but has non-cuban distribution: " + area.getIdInVocabulary() + "-" + status.getIdInVocabulary());
1254
                                logger.warn(state.getCurrentLine() +": Taxon ("+name+")is endemic but has non-cuban distribution: " + area.getIdInVocabulary() + "-" + status.getIdInVocabulary());
1158 1255
                                return;
1159 1256
                            }
1160 1257
                        }
......
1164 1261
        }
1165 1262
        if (!state.isEndemic() && ! hasExternalPresence){
1166 1263
            String name = taxon.getName().getTitleCache();
1167
            logger.error(state.getCurrentLine() +": Taxon ("+name+")is not endemic but has no non-cuban distribution" );
1264
            logger.error(state.getCurrentLine() +": Taxon ("+name+") is not endemic but has no non-cuban distribution" );
1168 1265
        }
1169 1266
    }
1170 1267

  
......
1406 1503
            TaxonNode familyTaxon,
1407 1504
            Taxon taxon) {
1408 1505

  
1506
        CubaImportConfigurator config = state.getConfig();
1507

  
1409 1508
        String famFRC = record.get("Fam. FRC");
1410 1509
        String famAS = record.get("Fam. A&S");
1411 1510
        String famFC = record.get("Fam. FC");
1511
        String famSanchez2017 = record.get("Fam. Sánchez 2017");
1512

  
1513
        if (config.isDoAltFlorasFRC()){
1514
            Reference refFRC = makeReference(state, CubaTransformer.uuidRefFRC);
1515
            makeSingleAlternativeFamily(state, taxon, famFRC, refFRC);
1516
        }
1517

  
1518
        if (config.isDoAltFlorasAS()){
1519
            Reference refAS = makeReference(state, CubaTransformer.uuidRefAS);
1520
            makeSingleAlternativeFamily(state, taxon, famAS, refAS);
1521
        }
1412 1522

  
1413
        Reference refFRC = makeReference(state, CubaTransformer.uuidRefFRC);
1414
        Reference refAS = makeReference(state, CubaTransformer.uuidRefAS);
1415
        Reference refFC = makeReference(state, CubaTransformer.uuidRefFC);
1523
        if (config.isDoAltFlorasFC()){
1524
            Reference refFC = makeReference(state, CubaTransformer.uuidRefFC);
1525
            makeSingleAlternativeFamily(state, taxon, famFC, refFC);
1526
        }
1416 1527

  
1417
        makeSingleAlternativeFamily(state, taxon, famFRC, refFRC);
1418
        makeSingleAlternativeFamily(state, taxon, famAS, refAS);
1419
        makeSingleAlternativeFamily(state, taxon, famFC, refFC);
1528
        if (config.isDoAltFlorasSanchez2017()){
1529
            Reference refSanchez2017 = makeReference(state, CubaTransformer.uuidRefSanchez);
1530
            makeSingleAlternativeFamily(state, taxon, famSanchez2017, refSanchez2017);
1531
        }
1420 1532
    }
1421 1533

  
1422 1534

  
......
1545 1657
        PresenceAbsenceTerm status = state.getTransformer().getPresenceTermByKey(statusStr);
1546 1658
        if (status == null){
1547 1659
//            PresenceAbsenceTerm highestStatus = state.getHighestStatusForTaxon();
1660
            if ("– 7".equals(statusStr)){
1661
                statusStr = "–";
1662
            }else if ("p 78".equals(statusStr)){
1663
                statusStr = "p";
1664
            }
1548 1665
            if (state.isCubanProvince() && isMinus(statusStr)){
1549 1666
//                getAbsenceTermForStatus(state, highestStatus);
1550 1667
                //we now handle cuban provinces same as external regions
......
1553 1670
                status = state.getTransformer().getPresenceTermByKey("--");
1554 1671
            }else{
1555 1672
//                logger.warn("Unhandled status str for provinces / external regions: " + statusStr);
1673

  
1556 1674
                UUID statusUuid = state.getTransformer().getPresenceTermUuid(statusStr);
1557 1675
                if (statusUuid == null){
1558
                    logger.error(state.getCurrentLine() + ": Undefined status str for provinces / external regions. No UUID given: '" + statusStr + "'");
1676
                    if (! ("78".equals(statusStr)|| "1".equals(statusStr)||"8".equals(statusStr))){
1677
                        logger.error(state.getCurrentLine() + ": Undefined status str for provinces / external regions. No UUID given: '" + statusStr + "'");
1678
                    }
1559 1679
                }else{
1560 1680
                    status = getPresenceTerm(state, statusUuid, statusStr, statusStr, statusStr, false);
1561 1681
                }
app-import/src/main/java/eu/etaxonomy/cdm/io/cuba/CubaImportConfigurator.java
10 10
package eu.etaxonomy.cdm.io.cuba;
11 11

  
12 12
import java.net.URI;
13
import java.util.UUID;
14 13

  
15 14
import org.apache.log4j.Logger;
16 15

  
......
36 35

  
37 36
    private String cubaReferenceTitle;
38 37

  
39
    private UUID uuidCyprusReference = UUID.fromString("b5281cd3-9d5d-4ae2-8d55-b62a592ce846");
40

  
41 38
	private boolean isDoTaxa;
42 39

  
40
	private boolean doAltFlorasSanchez2017;
41
    private boolean doAltFlorasFRC;
42
    private boolean doAltFlorasFC;
43
    private boolean doAltFlorasAS;
44

  
45

  
43 46
    private boolean doVocabularies;
44 47
    public void setDoVocabularies(boolean doVocabularies) {this.doVocabularies = doVocabularies;}
45 48
    public boolean isDoVocabularies() {return doVocabularies;}
......
99 102
	}
100 103

  
101 104

  
102
	public void setUuidCyprusReference(UUID uuidCyprusReference) {
103
		this.uuidCyprusReference = uuidCyprusReference;
104
	}
105

  
106

  
107
	public UUID getUuidCyprusReference() {
108
		return uuidCyprusReference;
109
	}
110

  
111

  
112 105
	public void setCubaReferenceTitle(String cyprusReferenceTitle) {
113 106
		this.cubaReferenceTitle = cyprusReferenceTitle;
114 107
	}
......
127 120
	}
128 121

  
129 122

  
123
	//************* ALT FLORAS ********************/
124

  
125
	public boolean isDoAltFlorasSanchez2017() {
126
        return doAltFlorasSanchez2017;
127
    }
128
    public void setDoAltFlorasSanchez2017(boolean doAltFlorasSanchez2017) {
129
        this.doAltFlorasSanchez2017 = doAltFlorasSanchez2017;
130
    }
131
    public boolean isDoAltFlorasFRC() {
132
        return doAltFlorasFRC;
133
    }
134
    public void setDoAltFlorasFRC(boolean doAltFlorasFRC) {
135
        this.doAltFlorasFRC = doAltFlorasFRC;
136
    }
137
    public boolean isDoAltFlorasFC() {
138
        return doAltFlorasFC;
139
    }
140
    public void setDoAltFlorasFC(boolean doAltFlorasFC) {
141
        this.doAltFlorasFC = doAltFlorasFC;
142
    }
143
    public boolean isDoAltFlorasAS() {
144
        return doAltFlorasAS;
145
    }
146
    public void setDoAltFlorasAS(boolean doAltFlorasAS) {
147
        this.doAltFlorasAS = doAltFlorasAS;
148
    }
149

  
150

  
130 151

  
131 152
}
app-import/src/main/java/eu/etaxonomy/cdm/io/cuba/CubaTransformer.java
32 32
    public static final UUID uuidRefFRC = UUID.fromString("c1caf6a2-5083-4f44-8f97-9abe23a84cd8");
33 33
    public static final UUID uuidRefAS = UUID.fromString("1f15291a-b4c5-4e15-960f-d0145a250539");
34 34
    public static final UUID uuidRefFC = UUID.fromString("c5a0bfb8-85b2-422d-babe-423aa2e24c35");
35
    public static final UUID uuidRefSanchez = UUID.fromString("d861aa65-9c56-4405-84c8-cdaa65b01f0b");
36

  
37
    //refPteridophyta
38
    protected static final UUID uuidRefPteridophyta78 = UUID.fromString("e9f68d98-c706-4bb6-8cbf-2be1f3897469");
39
    protected static final UUID uuidRefPteridophyta2 = UUID.fromString("910ae303-bd67-4dce-bb12-aebf889ca3fd");
40
    protected static final UUID uuidRefPteridophyta3 = UUID.fromString("80f83b9a-2aa4-4dfb-bbcd-457d3ed9e836");
41
    protected static final UUID uuidRefPteridophyta5 = UUID.fromString("bf41be7c-5832-4659-be47-66b94bf393f4");
42
    protected static final UUID uuidRefPteridophyta8_82 = UUID.fromString("21ca49a2-4243-4041-b876-13dfbcc7abb8");
43
    protected static final UUID uuidRefPteridophyta7 = UUID.fromString("251a51e2-8371-42e2-b042-063ecaaef241");
44

  
35 45

  
36 46
    //featureUUID
37 47
    public static final UUID uuidAlternativeFamily = UUID.fromString("a005f8a1-6377-4641-a826-185f67136860");

Also available in: Unified diff

Add picture from clipboard (Maximum size: 40 MB)