Project

General

Profile

« Previous | Next » 

Revision 55a5d551

Added by Andreas Müller almost 4 years ago

ref #9145 add IpniSourcesImport for E+M IPNI data

View differences:

app-import/src/main/java/eu/etaxonomy/cdm/app/common/CdmDestinations.java
188 188
        return CdmDestinations.makeDestination(dbType, cdmServer, cdmDB, port, cdmUserName, null);
189 189
    }
190 190

  
191
    public static ICdmDataSource cdm_local_euromed(){
192
        DatabaseTypeEnum dbType = DatabaseTypeEnum.MySQL;
193
        String cdmServer = "127.0.0.1";
194
        String cdmDB = "cdm_euromed";
195
        String cdmUserName = "edit";
196
        int port = 3306;
197
        return CdmDestinations.makeDestination(dbType, cdmServer, cdmDB, port, cdmUserName, null);
198
    }
199

  
191 200
    public static ICdmDataSource cdm_local_mysql_tmpTest(){
192 201
        DatabaseTypeEnum dbType = DatabaseTypeEnum.MySQL;
193 202
        String cdmServer = "127.0.0.1";
......
197 206
        return CdmDestinations.makeDestination(dbType, cdmServer, cdmDB, port, cdmUserName, null);
198 207
    }
199 208

  
200
	public static ICdmDataSource cdm_local_euromed(){
201
		DatabaseTypeEnum dbType = DatabaseTypeEnum.MySQL;
202
		String cdmServer = "127.0.0.1";
203
		String cdmDB = "euroMed";
204
		String cdmUserName = "edit";
205
		return makeDestination(dbType, cdmServer, cdmDB, -1, cdmUserName, null);
206
	}
207

  
208 209
	public static ICdmDataSource cdm_local_euromed2(){
209 210
		DatabaseTypeEnum dbType = DatabaseTypeEnum.MySQL;
210 211
		String cdmServer = "127.0.0.1";
app-import/src/main/java/eu/etaxonomy/cdm/app/euromed/EuroMedIpniDistributionSourcesImportActivator.java
1
/**
2
* Copyright (C) 2015 EDIT
3
* European Distributed Institute of Taxonomy
4
* http://www.e-taxonomy.eu
5
*
6
* The contents of this file are subject to the Mozilla Public License Version 1.1
7
* See LICENSE.TXT at the top of this package for the full license terms.
8
*/
9
package eu.etaxonomy.cdm.app.euromed;
10

  
11
import java.net.URI;
12
import java.util.UUID;
13

  
14
import org.apache.log4j.Logger;
15

  
16
import eu.etaxonomy.cdm.app.common.CdmDestinations;
17
import eu.etaxonomy.cdm.database.DbSchemaValidation;
18
import eu.etaxonomy.cdm.database.ICdmDataSource;
19
import eu.etaxonomy.cdm.io.common.CdmDefaultImport;
20
import eu.etaxonomy.cdm.io.common.IImportConfigurator.CHECK;
21
import eu.etaxonomy.cdm.io.common.ImportResult;
22
import eu.etaxonomy.cdm.io.euromed.IpniSourcesImportConfigurator;
23
import eu.etaxonomy.cdm.model.common.VerbatimTimePeriod;
24
import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
25
import eu.etaxonomy.cdm.model.reference.Reference;
26
import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
27

  
28
/**
29
 * @author a.mueller
30
 * @since 18.10.2019
31
 */
32
public class EuroMedIpniDistributionSourcesImportActivator {
33

  
34
    @SuppressWarnings("unused")
35
    private static final Logger logger = Logger.getLogger(EuroMedIpniDistributionSourcesImportActivator.class);
36

  
37
    //database validation status (create, update, validate ...)
38
    static DbSchemaValidation dbSchemaValidation = DbSchemaValidation.VALIDATE;
39

  
40
    static final ICdmDataSource cdmDestination = CdmDestinations.cdm_local_euromed();
41
//   static final ICdmDataSource cdmDestination = CdmDestinations.cdm_production_euromed();
42

  
43
    //check - import
44
    static CHECK check = CHECK.IMPORT_WITHOUT_CHECK;
45

  
46
    private void doImport(ICdmDataSource cdmDestination){
47

  
48
        URI source = getEM_IpniUri();
49
        IpniSourcesImportConfigurator config;
50
//        try {
51
            config = IpniSourcesImportConfigurator.NewInstance(source, cdmDestination);
52

  
53
            config.setDbSchemaValidation(dbSchemaValidation);
54
            config.setCheck(check);
55
            config.setNomenclaturalCode(NomenclaturalCode.ICNAFP);
56
            config.setSourceReference(getSourceReference());
57
//            config.setAddAuthorsToReference(addAuthorsToReferences);
58

  
59
            CdmDefaultImport<IpniSourcesImportConfigurator> myImport = new CdmDefaultImport<>();
60
            ImportResult result = myImport.invoke(config);
61
            System.out.println(result.createReport());
62
//        } catch (IOException e) {
63
//            System.out.println("URI not 'found': " + source);
64
//        }
65
    }
66

  
67
    private URI getEM_IpniUri(){
68
        String fileName = "IPNI-name-EM-import.xlsx";
69

  
70
        URI uri = URI.create("file:////BGBM-PESIHPC/EuroMed/" +  fileName);
71
        return uri;
72
    }
73

  
74
    private Reference getSourceReference(){
75
        Reference result = ReferenceFactory.newDatabase();
76
        result.setTitle("Euro+Med IPNI import");
77
        result.setUuid(UUID.fromString("937f3a38-dbb4-4888-9ea3-2f52d47f7953"));
78
        result.setDatePublished(VerbatimTimePeriod.NewVerbatimNowInstance());
79
        return result;
80
    }
81

  
82
    public static void main(String[] args) {
83
        EuroMedIpniDistributionSourcesImportActivator me = new EuroMedIpniDistributionSourcesImportActivator();
84
        me.doImport(cdmDestination);
85
        System.exit(0);
86
    }
87
}
app-import/src/main/java/eu/etaxonomy/cdm/io/euromed/IpniSourcesImport.java
1
/**
2
* Copyright (C) 2017 EDIT
3
* European Distributed Institute of Taxonomy
4
* http://www.e-taxonomy.eu
5
*
6
* The contents of this file are subject to the Mozilla Public License Version 1.1
7
* See LICENSE.TXT at the top of this package for the full license terms.
8
*/
9
package eu.etaxonomy.cdm.io.euromed;
10

  
11
import java.util.ArrayList;
12
import java.util.HashMap;
13
import java.util.List;
14
import java.util.Map;
15
import java.util.Set;
16

  
17
import org.apache.log4j.Logger;
18
import org.springframework.stereotype.Component;
19
import org.springframework.transaction.TransactionStatus;
20

  
21
import eu.etaxonomy.cdm.api.service.pager.Pager;
22
import eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer;
23
import eu.etaxonomy.cdm.io.mexico.SimpleExcelTaxonImport;
24
import eu.etaxonomy.cdm.io.mexico.SimpleExcelTaxonImportState;
25
import eu.etaxonomy.cdm.model.description.DescriptionElementSource;
26
import eu.etaxonomy.cdm.model.description.Distribution;
27
import eu.etaxonomy.cdm.model.description.Feature;
28
import eu.etaxonomy.cdm.model.location.NamedArea;
29
import eu.etaxonomy.cdm.model.name.Rank;
30
import eu.etaxonomy.cdm.model.reference.OriginalSourceType;
31
import eu.etaxonomy.cdm.model.taxon.Taxon;
32
import eu.etaxonomy.cdm.model.term.TermVocabulary;
33

  
34
/**
35
 * This import adds the distribution source to data imported via E+M IPNI new names import.
36
 *
37
 * @author a.mueller
38
 * @since 10.07.2020
39
 */
40
@Component
41
public class IpniSourcesImport<CONFIG extends IpniSourcesImportConfigurator>
42
        extends SimpleExcelTaxonImport<CONFIG> {
43

  
44
    private static final long serialVersionUID = -6723116237971852295L;
45

  
46
    private static final Logger logger = Logger.getLogger(IpniSourcesImport.class);
47

  
48
    private static final String NAMECACHE = "full_name_without_family_and_authors";
49
    private static final String GENUS = "genus";
50
    private static final String SPECIES = "species";
51
    private static final String INFRA_SPECIES = "infraspecies";
52
    private static final String RANK = "rank";
53
    private static final String AUTHORS = "EMauthors";
54
    private static final String EM_GEO = "EM-geo";
55

  
56
    private Map<String,NamedArea> areaMap;
57

  
58
    @Override
59
    protected String getWorksheetName(CONFIG config) {
60
        return "_11_IPNI_name_w_EM_genus_tax_m2";
61
    }
62

  
63
    private boolean isFirst = true;
64
    private TransactionStatus tx = null;
65

  
66
    @Override
67
    protected void firstPass(SimpleExcelTaxonImportState<CONFIG> state) {
68
        if (isFirst){
69
            tx = this.startTransaction();
70
            isFirst = false;
71
        }
72
        getAreaMap();
73

  
74
        String line = state.getCurrentLine() + ": ";
75
        Map<String, String> record = state.getOriginalRecord();
76

  
77
        String uninomial = getValue(record, GENUS);
78
        String specificEpithet = getValue(record, SPECIES);
79
        String infraspecificEpithet = getValue(record, INFRA_SPECIES);
80
        String nameCache = getValue(record, NAMECACHE);
81
        List<String> propertyPaths = null;
82
        String authorshipCache = "*";
83
        Rank rank = getRank(state);
84
        Pager<Taxon> taxonPager = getTaxonService().findTaxaByName(Taxon.class, uninomial, "*", specificEpithet,
85
                infraspecificEpithet, authorshipCache, rank, null, null, propertyPaths);
86
        List<Taxon> taxa = getPublishedTaxa(taxonPager, line);
87
        if (taxa.size() == 1){
88
            Taxon taxon = taxa.get(0);
89
            makeDistributionSource(state, line, taxon);
90
        }else{
91
            logger.warn(line + "Could not find unique taxon. Count: " + taxa.size() +"; "  + nameCache);
92
        }
93

  
94
    }
95

  
96
    private List<Taxon> getPublishedTaxa(Pager<Taxon> taxonPager, String line) {
97
        List<Taxon> result = new ArrayList<>();
98
        for (Taxon taxon : taxonPager.getRecords()){
99
            if (!taxon.isPublish()){
100
                logger.warn(line + "Unpublished taxon exists: " +  taxon.getTitleCache());
101
            }else{
102
                result.add(taxon);
103
            }
104
        }
105
        return result;
106
    }
107

  
108
    private void makeDistributionSource(SimpleExcelTaxonImportState<CONFIG> state, String line, Taxon taxon) {
109

  
110
        Set<Distribution> distributions = taxon.getDescriptionItems(Feature.DISTRIBUTION(), Distribution.class);
111

  
112
        //single areas
113
        Map<String, String> record = state.getOriginalRecord();
114
        String allAreaStr = getValue(record, EM_GEO);
115
        if(isBlank(allAreaStr)){
116
            logger.warn(line + "No distribution data exists in IPNI file: " + taxon.getName().getTitleCache());
117
        }else{
118
            NamedArea emArea = getAreaMap().get("EM");
119
            handleArea(line, taxon, distributions, emArea);
120

  
121
            String[] areaSplit = allAreaStr.split(",");
122
            for (String areaStr: areaSplit){
123
                NamedArea area = getAreaMap().get(areaStr.trim());
124
                handleArea(line, taxon, distributions, area);
125
            }
126
        }
127
    }
128

  
129
    /**
130
     * @param line
131
     * @param taxon
132
     * @param distributions
133
     * @param area
134
     */
135
    private void handleArea(String line, Taxon taxon, Set<Distribution> distributions, NamedArea area) {
136
        Distribution distribution = findDistribution(distributions, area);
137
        if (distribution == null){
138
            logger.warn("line + Distribution not found: " + taxon.getName().getTitleCache() + ":" + area.getTitleCache());
139
        }else{
140
            if (distribution.getSources().isEmpty()){
141
                addNomenclaturalSource(taxon, distribution, line);
142
            }else{
143
                logger.warn(line + "Distribution has source already: " + taxon.getName().getTitleCache() + ": " + area.getTitleCache() + "; Source(s): " + getSourceString(distribution.getSources()));
144
            }
145
        }
146
    }
147

  
148
    private String getSourceString(Set<DescriptionElementSource> sources) {
149
        String result = "";
150
        for (DescriptionElementSource source : sources){
151
            if (source.getCitation() != null){
152
                result += source.getCitation().getTitleCache();
153
            }else{
154
                result += "--source has no citation --";
155
            }
156
        }
157
        return result;
158
    }
159

  
160
    private void addNomenclaturalSource(Taxon taxon, Distribution distribution, String line) {
161
        distribution.addSource(OriginalSourceType.PrimaryTaxonomicSource,
162
                null, null, taxon.getName().getNomenclaturalReference(),
163
                taxon.getName().getNomenclaturalMicroReference(),
164
                taxon.getName(), null);
165
        if(taxon.getName().getNomenclaturalReference() == null){
166
            logger.warn(line + "No nomenclatural source available" + taxon.getName().getTitleCache());
167
        }
168
    }
169

  
170
    private Distribution findDistribution(Set<Distribution> distributions, NamedArea area) {
171
        Distribution result = null;
172
        for (Distribution distribution : distributions){
173
            if (area.equals(distribution.getArea())){
174
                result = distribution;
175
            }
176
        }
177
        return result;
178
    }
179

  
180
    private Map<String, NamedArea> getAreaMap() {
181
        if (areaMap == null){
182
            makeAreaMap();
183
        }
184
        return areaMap;
185
    }
186

  
187
    private void makeAreaMap() {
188
        areaMap = new HashMap<>();
189
        @SuppressWarnings("unchecked")
190
        TermVocabulary<NamedArea> emAreaVoc = getVocabularyService().find(BerlinModelTransformer.uuidVocEuroMedAreas);
191
        for (NamedArea area: emAreaVoc.getTerms()){
192
            areaMap.put(area.getIdInVocabulary(), area);
193
        }
194
    }
195

  
196
    private Rank getRank(SimpleExcelTaxonImportState<CONFIG> state) {
197
        Map<String, String> record = state.getOriginalRecord();
198
        String rankStr = getValue(record, RANK);
199
        if ("spec.".equals(rankStr)){
200
            return Rank.SPECIES();
201
        }else if ("subsp.".equals(rankStr)){
202
            return Rank.SUBSPECIES();
203
        }else{
204
            logger.warn("Unknown rank: " + rankStr);
205
            return null;
206
        }
207
    }
208

  
209
    @Override
210
    protected void secondPass(SimpleExcelTaxonImportState<CONFIG> state) {
211
        if (tx != null){
212
            this.commitTransaction(tx);
213
            tx = null;
214
        }
215
    }
216
}
app-import/src/main/java/eu/etaxonomy/cdm/io/euromed/IpniSourcesImportConfigurator.java
1
/**
2
* Copyright (C) 2017 EDIT
3
* European Distributed Institute of Taxonomy
4
* http://www.e-taxonomy.eu
5
*
6
* The contents of this file are subject to the Mozilla Public License Version 1.1
7
* See LICENSE.TXT at the top of this package for the full license terms.
8
*/
9
package eu.etaxonomy.cdm.io.euromed;
10

  
11
import java.net.URI;
12

  
13
import eu.etaxonomy.cdm.database.ICdmDataSource;
14
import eu.etaxonomy.cdm.io.common.ImportStateBase;
15
import eu.etaxonomy.cdm.io.excel.common.ExcelImportConfiguratorBase;
16
import eu.etaxonomy.cdm.io.mexico.SimpleExcelTaxonImportState;
17
import eu.etaxonomy.cdm.model.name.NomenclaturalCode;
18

  
19
/**
20
 * @author a.mueller
21
 * @since 23.10.2019
22
 */
23
public class IpniSourcesImportConfigurator extends ExcelImportConfiguratorBase {
24

  
25
    private static final long serialVersionUID = 7883220110057878974L;
26

  
27
    public static IpniSourcesImportConfigurator NewInstance(URI uri, ICdmDataSource cdmDestination) {
28
        return new IpniSourcesImportConfigurator(uri, cdmDestination);
29
    }
30

  
31
    private IpniSourcesImportConfigurator(URI uri, ICdmDataSource cdmDestination) {
32
        super(uri, cdmDestination, null);
33
        this.setNomenclaturalCode(NomenclaturalCode.ICNAFP);
34
    }
35

  
36
    @SuppressWarnings("unchecked")
37
    @Override
38
    protected void makeIoClassList() {
39
        ioClassList = new Class[]{
40
                IpniSourcesImport.class
41
            };
42
    }
43

  
44
    @Override
45
    public ImportStateBase getNewState() {
46
        return new SimpleExcelTaxonImportState<>(this);
47
    }
48
//
49
//    /**
50
//     * If <code>true</code> the name authors will be added
51
//     * to the nomenclatural reference (Book or Article) though
52
//     * it might not be the exact same author.<BR>
53
//     * Default is <code>true</code>
54
//     */
55
//    public boolean isAddAuthorsToReference() {
56
//        return addAuthorsToReference;
57
//    }
58
//    /**
59
//     * @see #isAddAuthorsToReference()
60
//     */
61
//    public void setAddAuthorsToReference(boolean addAuthorsToReference) {
62
//        this.addAuthorsToReference = addAuthorsToReference;
63
//    }
64
}

Also available in: Unified diff