Revision 55a5d551
Added by Andreas Müller about 3 years ago
app-import/src/main/java/eu/etaxonomy/cdm/app/common/CdmDestinations.java | ||
---|---|---|
188 | 188 |
return CdmDestinations.makeDestination(dbType, cdmServer, cdmDB, port, cdmUserName, null); |
189 | 189 |
} |
190 | 190 |
|
191 |
public static ICdmDataSource cdm_local_euromed(){ |
|
192 |
DatabaseTypeEnum dbType = DatabaseTypeEnum.MySQL; |
|
193 |
String cdmServer = "127.0.0.1"; |
|
194 |
String cdmDB = "cdm_euromed"; |
|
195 |
String cdmUserName = "edit"; |
|
196 |
int port = 3306; |
|
197 |
return CdmDestinations.makeDestination(dbType, cdmServer, cdmDB, port, cdmUserName, null); |
|
198 |
} |
|
199 |
|
|
191 | 200 |
public static ICdmDataSource cdm_local_mysql_tmpTest(){ |
192 | 201 |
DatabaseTypeEnum dbType = DatabaseTypeEnum.MySQL; |
193 | 202 |
String cdmServer = "127.0.0.1"; |
... | ... | |
197 | 206 |
return CdmDestinations.makeDestination(dbType, cdmServer, cdmDB, port, cdmUserName, null); |
198 | 207 |
} |
199 | 208 |
|
200 |
public static ICdmDataSource cdm_local_euromed(){ |
|
201 |
DatabaseTypeEnum dbType = DatabaseTypeEnum.MySQL; |
|
202 |
String cdmServer = "127.0.0.1"; |
|
203 |
String cdmDB = "euroMed"; |
|
204 |
String cdmUserName = "edit"; |
|
205 |
return makeDestination(dbType, cdmServer, cdmDB, -1, cdmUserName, null); |
|
206 |
} |
|
207 |
|
|
208 | 209 |
public static ICdmDataSource cdm_local_euromed2(){ |
209 | 210 |
DatabaseTypeEnum dbType = DatabaseTypeEnum.MySQL; |
210 | 211 |
String cdmServer = "127.0.0.1"; |
app-import/src/main/java/eu/etaxonomy/cdm/app/euromed/EuroMedIpniDistributionSourcesImportActivator.java | ||
---|---|---|
1 |
/** |
|
2 |
* Copyright (C) 2015 EDIT |
|
3 |
* European Distributed Institute of Taxonomy |
|
4 |
* http://www.e-taxonomy.eu |
|
5 |
* |
|
6 |
* The contents of this file are subject to the Mozilla Public License Version 1.1 |
|
7 |
* See LICENSE.TXT at the top of this package for the full license terms. |
|
8 |
*/ |
|
9 |
package eu.etaxonomy.cdm.app.euromed; |
|
10 |
|
|
11 |
import java.net.URI; |
|
12 |
import java.util.UUID; |
|
13 |
|
|
14 |
import org.apache.log4j.Logger; |
|
15 |
|
|
16 |
import eu.etaxonomy.cdm.app.common.CdmDestinations; |
|
17 |
import eu.etaxonomy.cdm.database.DbSchemaValidation; |
|
18 |
import eu.etaxonomy.cdm.database.ICdmDataSource; |
|
19 |
import eu.etaxonomy.cdm.io.common.CdmDefaultImport; |
|
20 |
import eu.etaxonomy.cdm.io.common.IImportConfigurator.CHECK; |
|
21 |
import eu.etaxonomy.cdm.io.common.ImportResult; |
|
22 |
import eu.etaxonomy.cdm.io.euromed.IpniSourcesImportConfigurator; |
|
23 |
import eu.etaxonomy.cdm.model.common.VerbatimTimePeriod; |
|
24 |
import eu.etaxonomy.cdm.model.name.NomenclaturalCode; |
|
25 |
import eu.etaxonomy.cdm.model.reference.Reference; |
|
26 |
import eu.etaxonomy.cdm.model.reference.ReferenceFactory; |
|
27 |
|
|
28 |
/** |
|
29 |
* @author a.mueller |
|
30 |
* @since 18.10.2019 |
|
31 |
*/ |
|
32 |
public class EuroMedIpniDistributionSourcesImportActivator { |
|
33 |
|
|
34 |
@SuppressWarnings("unused") |
|
35 |
private static final Logger logger = Logger.getLogger(EuroMedIpniDistributionSourcesImportActivator.class); |
|
36 |
|
|
37 |
//database validation status (create, update, validate ...) |
|
38 |
static DbSchemaValidation dbSchemaValidation = DbSchemaValidation.VALIDATE; |
|
39 |
|
|
40 |
static final ICdmDataSource cdmDestination = CdmDestinations.cdm_local_euromed(); |
|
41 |
// static final ICdmDataSource cdmDestination = CdmDestinations.cdm_production_euromed(); |
|
42 |
|
|
43 |
//check - import |
|
44 |
static CHECK check = CHECK.IMPORT_WITHOUT_CHECK; |
|
45 |
|
|
46 |
private void doImport(ICdmDataSource cdmDestination){ |
|
47 |
|
|
48 |
URI source = getEM_IpniUri(); |
|
49 |
IpniSourcesImportConfigurator config; |
|
50 |
// try { |
|
51 |
config = IpniSourcesImportConfigurator.NewInstance(source, cdmDestination); |
|
52 |
|
|
53 |
config.setDbSchemaValidation(dbSchemaValidation); |
|
54 |
config.setCheck(check); |
|
55 |
config.setNomenclaturalCode(NomenclaturalCode.ICNAFP); |
|
56 |
config.setSourceReference(getSourceReference()); |
|
57 |
// config.setAddAuthorsToReference(addAuthorsToReferences); |
|
58 |
|
|
59 |
CdmDefaultImport<IpniSourcesImportConfigurator> myImport = new CdmDefaultImport<>(); |
|
60 |
ImportResult result = myImport.invoke(config); |
|
61 |
System.out.println(result.createReport()); |
|
62 |
// } catch (IOException e) { |
|
63 |
// System.out.println("URI not 'found': " + source); |
|
64 |
// } |
|
65 |
} |
|
66 |
|
|
67 |
private URI getEM_IpniUri(){ |
|
68 |
String fileName = "IPNI-name-EM-import.xlsx"; |
|
69 |
|
|
70 |
URI uri = URI.create("file:////BGBM-PESIHPC/EuroMed/" + fileName); |
|
71 |
return uri; |
|
72 |
} |
|
73 |
|
|
74 |
private Reference getSourceReference(){ |
|
75 |
Reference result = ReferenceFactory.newDatabase(); |
|
76 |
result.setTitle("Euro+Med IPNI import"); |
|
77 |
result.setUuid(UUID.fromString("937f3a38-dbb4-4888-9ea3-2f52d47f7953")); |
|
78 |
result.setDatePublished(VerbatimTimePeriod.NewVerbatimNowInstance()); |
|
79 |
return result; |
|
80 |
} |
|
81 |
|
|
82 |
public static void main(String[] args) { |
|
83 |
EuroMedIpniDistributionSourcesImportActivator me = new EuroMedIpniDistributionSourcesImportActivator(); |
|
84 |
me.doImport(cdmDestination); |
|
85 |
System.exit(0); |
|
86 |
} |
|
87 |
} |
app-import/src/main/java/eu/etaxonomy/cdm/io/euromed/IpniSourcesImport.java | ||
---|---|---|
1 |
/** |
|
2 |
* Copyright (C) 2017 EDIT |
|
3 |
* European Distributed Institute of Taxonomy |
|
4 |
* http://www.e-taxonomy.eu |
|
5 |
* |
|
6 |
* The contents of this file are subject to the Mozilla Public License Version 1.1 |
|
7 |
* See LICENSE.TXT at the top of this package for the full license terms. |
|
8 |
*/ |
|
9 |
package eu.etaxonomy.cdm.io.euromed; |
|
10 |
|
|
11 |
import java.util.ArrayList; |
|
12 |
import java.util.HashMap; |
|
13 |
import java.util.List; |
|
14 |
import java.util.Map; |
|
15 |
import java.util.Set; |
|
16 |
|
|
17 |
import org.apache.log4j.Logger; |
|
18 |
import org.springframework.stereotype.Component; |
|
19 |
import org.springframework.transaction.TransactionStatus; |
|
20 |
|
|
21 |
import eu.etaxonomy.cdm.api.service.pager.Pager; |
|
22 |
import eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer; |
|
23 |
import eu.etaxonomy.cdm.io.mexico.SimpleExcelTaxonImport; |
|
24 |
import eu.etaxonomy.cdm.io.mexico.SimpleExcelTaxonImportState; |
|
25 |
import eu.etaxonomy.cdm.model.description.DescriptionElementSource; |
|
26 |
import eu.etaxonomy.cdm.model.description.Distribution; |
|
27 |
import eu.etaxonomy.cdm.model.description.Feature; |
|
28 |
import eu.etaxonomy.cdm.model.location.NamedArea; |
|
29 |
import eu.etaxonomy.cdm.model.name.Rank; |
|
30 |
import eu.etaxonomy.cdm.model.reference.OriginalSourceType; |
|
31 |
import eu.etaxonomy.cdm.model.taxon.Taxon; |
|
32 |
import eu.etaxonomy.cdm.model.term.TermVocabulary; |
|
33 |
|
|
34 |
/** |
|
35 |
* This import adds the distribution source to data imported via E+M IPNI new names import. |
|
36 |
* |
|
37 |
* @author a.mueller |
|
38 |
* @since 10.07.2020 |
|
39 |
*/ |
|
40 |
@Component |
|
41 |
public class IpniSourcesImport<CONFIG extends IpniSourcesImportConfigurator> |
|
42 |
extends SimpleExcelTaxonImport<CONFIG> { |
|
43 |
|
|
44 |
private static final long serialVersionUID = -6723116237971852295L; |
|
45 |
|
|
46 |
private static final Logger logger = Logger.getLogger(IpniSourcesImport.class); |
|
47 |
|
|
48 |
private static final String NAMECACHE = "full_name_without_family_and_authors"; |
|
49 |
private static final String GENUS = "genus"; |
|
50 |
private static final String SPECIES = "species"; |
|
51 |
private static final String INFRA_SPECIES = "infraspecies"; |
|
52 |
private static final String RANK = "rank"; |
|
53 |
private static final String AUTHORS = "EMauthors"; |
|
54 |
private static final String EM_GEO = "EM-geo"; |
|
55 |
|
|
56 |
private Map<String,NamedArea> areaMap; |
|
57 |
|
|
58 |
@Override |
|
59 |
protected String getWorksheetName(CONFIG config) { |
|
60 |
return "_11_IPNI_name_w_EM_genus_tax_m2"; |
|
61 |
} |
|
62 |
|
|
63 |
private boolean isFirst = true; |
|
64 |
private TransactionStatus tx = null; |
|
65 |
|
|
66 |
@Override |
|
67 |
protected void firstPass(SimpleExcelTaxonImportState<CONFIG> state) { |
|
68 |
if (isFirst){ |
|
69 |
tx = this.startTransaction(); |
|
70 |
isFirst = false; |
|
71 |
} |
|
72 |
getAreaMap(); |
|
73 |
|
|
74 |
String line = state.getCurrentLine() + ": "; |
|
75 |
Map<String, String> record = state.getOriginalRecord(); |
|
76 |
|
|
77 |
String uninomial = getValue(record, GENUS); |
|
78 |
String specificEpithet = getValue(record, SPECIES); |
|
79 |
String infraspecificEpithet = getValue(record, INFRA_SPECIES); |
|
80 |
String nameCache = getValue(record, NAMECACHE); |
|
81 |
List<String> propertyPaths = null; |
|
82 |
String authorshipCache = "*"; |
|
83 |
Rank rank = getRank(state); |
|
84 |
Pager<Taxon> taxonPager = getTaxonService().findTaxaByName(Taxon.class, uninomial, "*", specificEpithet, |
|
85 |
infraspecificEpithet, authorshipCache, rank, null, null, propertyPaths); |
|
86 |
List<Taxon> taxa = getPublishedTaxa(taxonPager, line); |
|
87 |
if (taxa.size() == 1){ |
|
88 |
Taxon taxon = taxa.get(0); |
|
89 |
makeDistributionSource(state, line, taxon); |
|
90 |
}else{ |
|
91 |
logger.warn(line + "Could not find unique taxon. Count: " + taxa.size() +"; " + nameCache); |
|
92 |
} |
|
93 |
|
|
94 |
} |
|
95 |
|
|
96 |
private List<Taxon> getPublishedTaxa(Pager<Taxon> taxonPager, String line) { |
|
97 |
List<Taxon> result = new ArrayList<>(); |
|
98 |
for (Taxon taxon : taxonPager.getRecords()){ |
|
99 |
if (!taxon.isPublish()){ |
|
100 |
logger.warn(line + "Unpublished taxon exists: " + taxon.getTitleCache()); |
|
101 |
}else{ |
|
102 |
result.add(taxon); |
|
103 |
} |
|
104 |
} |
|
105 |
return result; |
|
106 |
} |
|
107 |
|
|
108 |
private void makeDistributionSource(SimpleExcelTaxonImportState<CONFIG> state, String line, Taxon taxon) { |
|
109 |
|
|
110 |
Set<Distribution> distributions = taxon.getDescriptionItems(Feature.DISTRIBUTION(), Distribution.class); |
|
111 |
|
|
112 |
//single areas |
|
113 |
Map<String, String> record = state.getOriginalRecord(); |
|
114 |
String allAreaStr = getValue(record, EM_GEO); |
|
115 |
if(isBlank(allAreaStr)){ |
|
116 |
logger.warn(line + "No distribution data exists in IPNI file: " + taxon.getName().getTitleCache()); |
|
117 |
}else{ |
|
118 |
NamedArea emArea = getAreaMap().get("EM"); |
|
119 |
handleArea(line, taxon, distributions, emArea); |
|
120 |
|
|
121 |
String[] areaSplit = allAreaStr.split(","); |
|
122 |
for (String areaStr: areaSplit){ |
|
123 |
NamedArea area = getAreaMap().get(areaStr.trim()); |
|
124 |
handleArea(line, taxon, distributions, area); |
|
125 |
} |
|
126 |
} |
|
127 |
} |
|
128 |
|
|
129 |
/** |
|
130 |
* @param line |
|
131 |
* @param taxon |
|
132 |
* @param distributions |
|
133 |
* @param area |
|
134 |
*/ |
|
135 |
private void handleArea(String line, Taxon taxon, Set<Distribution> distributions, NamedArea area) { |
|
136 |
Distribution distribution = findDistribution(distributions, area); |
|
137 |
if (distribution == null){ |
|
138 |
logger.warn("line + Distribution not found: " + taxon.getName().getTitleCache() + ":" + area.getTitleCache()); |
|
139 |
}else{ |
|
140 |
if (distribution.getSources().isEmpty()){ |
|
141 |
addNomenclaturalSource(taxon, distribution, line); |
|
142 |
}else{ |
|
143 |
logger.warn(line + "Distribution has source already: " + taxon.getName().getTitleCache() + ": " + area.getTitleCache() + "; Source(s): " + getSourceString(distribution.getSources())); |
|
144 |
} |
|
145 |
} |
|
146 |
} |
|
147 |
|
|
148 |
private String getSourceString(Set<DescriptionElementSource> sources) { |
|
149 |
String result = ""; |
|
150 |
for (DescriptionElementSource source : sources){ |
|
151 |
if (source.getCitation() != null){ |
|
152 |
result += source.getCitation().getTitleCache(); |
|
153 |
}else{ |
|
154 |
result += "--source has no citation --"; |
|
155 |
} |
|
156 |
} |
|
157 |
return result; |
|
158 |
} |
|
159 |
|
|
160 |
private void addNomenclaturalSource(Taxon taxon, Distribution distribution, String line) { |
|
161 |
distribution.addSource(OriginalSourceType.PrimaryTaxonomicSource, |
|
162 |
null, null, taxon.getName().getNomenclaturalReference(), |
|
163 |
taxon.getName().getNomenclaturalMicroReference(), |
|
164 |
taxon.getName(), null); |
|
165 |
if(taxon.getName().getNomenclaturalReference() == null){ |
|
166 |
logger.warn(line + "No nomenclatural source available" + taxon.getName().getTitleCache()); |
|
167 |
} |
|
168 |
} |
|
169 |
|
|
170 |
private Distribution findDistribution(Set<Distribution> distributions, NamedArea area) { |
|
171 |
Distribution result = null; |
|
172 |
for (Distribution distribution : distributions){ |
|
173 |
if (area.equals(distribution.getArea())){ |
|
174 |
result = distribution; |
|
175 |
} |
|
176 |
} |
|
177 |
return result; |
|
178 |
} |
|
179 |
|
|
180 |
private Map<String, NamedArea> getAreaMap() { |
|
181 |
if (areaMap == null){ |
|
182 |
makeAreaMap(); |
|
183 |
} |
|
184 |
return areaMap; |
|
185 |
} |
|
186 |
|
|
187 |
private void makeAreaMap() { |
|
188 |
areaMap = new HashMap<>(); |
|
189 |
@SuppressWarnings("unchecked") |
|
190 |
TermVocabulary<NamedArea> emAreaVoc = getVocabularyService().find(BerlinModelTransformer.uuidVocEuroMedAreas); |
|
191 |
for (NamedArea area: emAreaVoc.getTerms()){ |
|
192 |
areaMap.put(area.getIdInVocabulary(), area); |
|
193 |
} |
|
194 |
} |
|
195 |
|
|
196 |
private Rank getRank(SimpleExcelTaxonImportState<CONFIG> state) { |
|
197 |
Map<String, String> record = state.getOriginalRecord(); |
|
198 |
String rankStr = getValue(record, RANK); |
|
199 |
if ("spec.".equals(rankStr)){ |
|
200 |
return Rank.SPECIES(); |
|
201 |
}else if ("subsp.".equals(rankStr)){ |
|
202 |
return Rank.SUBSPECIES(); |
|
203 |
}else{ |
|
204 |
logger.warn("Unknown rank: " + rankStr); |
|
205 |
return null; |
|
206 |
} |
|
207 |
} |
|
208 |
|
|
209 |
@Override |
|
210 |
protected void secondPass(SimpleExcelTaxonImportState<CONFIG> state) { |
|
211 |
if (tx != null){ |
|
212 |
this.commitTransaction(tx); |
|
213 |
tx = null; |
|
214 |
} |
|
215 |
} |
|
216 |
} |
app-import/src/main/java/eu/etaxonomy/cdm/io/euromed/IpniSourcesImportConfigurator.java | ||
---|---|---|
1 |
/** |
|
2 |
* Copyright (C) 2017 EDIT |
|
3 |
* European Distributed Institute of Taxonomy |
|
4 |
* http://www.e-taxonomy.eu |
|
5 |
* |
|
6 |
* The contents of this file are subject to the Mozilla Public License Version 1.1 |
|
7 |
* See LICENSE.TXT at the top of this package for the full license terms. |
|
8 |
*/ |
|
9 |
package eu.etaxonomy.cdm.io.euromed; |
|
10 |
|
|
11 |
import java.net.URI; |
|
12 |
|
|
13 |
import eu.etaxonomy.cdm.database.ICdmDataSource; |
|
14 |
import eu.etaxonomy.cdm.io.common.ImportStateBase; |
|
15 |
import eu.etaxonomy.cdm.io.excel.common.ExcelImportConfiguratorBase; |
|
16 |
import eu.etaxonomy.cdm.io.mexico.SimpleExcelTaxonImportState; |
|
17 |
import eu.etaxonomy.cdm.model.name.NomenclaturalCode; |
|
18 |
|
|
19 |
/** |
|
20 |
* @author a.mueller |
|
21 |
* @since 23.10.2019 |
|
22 |
*/ |
|
23 |
public class IpniSourcesImportConfigurator extends ExcelImportConfiguratorBase { |
|
24 |
|
|
25 |
private static final long serialVersionUID = 7883220110057878974L; |
|
26 |
|
|
27 |
public static IpniSourcesImportConfigurator NewInstance(URI uri, ICdmDataSource cdmDestination) { |
|
28 |
return new IpniSourcesImportConfigurator(uri, cdmDestination); |
|
29 |
} |
|
30 |
|
|
31 |
private IpniSourcesImportConfigurator(URI uri, ICdmDataSource cdmDestination) { |
|
32 |
super(uri, cdmDestination, null); |
|
33 |
this.setNomenclaturalCode(NomenclaturalCode.ICNAFP); |
|
34 |
} |
|
35 |
|
|
36 |
@SuppressWarnings("unchecked") |
|
37 |
@Override |
|
38 |
protected void makeIoClassList() { |
|
39 |
ioClassList = new Class[]{ |
|
40 |
IpniSourcesImport.class |
|
41 |
}; |
|
42 |
} |
|
43 |
|
|
44 |
@Override |
|
45 |
public ImportStateBase getNewState() { |
|
46 |
return new SimpleExcelTaxonImportState<>(this); |
|
47 |
} |
|
48 |
// |
|
49 |
// /** |
|
50 |
// * If <code>true</code> the name authors will be added |
|
51 |
// * to the nomenclatural reference (Book or Article) though |
|
52 |
// * it might not be the exact same author.<BR> |
|
53 |
// * Default is <code>true</code> |
|
54 |
// */ |
|
55 |
// public boolean isAddAuthorsToReference() { |
|
56 |
// return addAuthorsToReference; |
|
57 |
// } |
|
58 |
// /** |
|
59 |
// * @see #isAddAuthorsToReference() |
|
60 |
// */ |
|
61 |
// public void setAddAuthorsToReference(boolean addAuthorsToReference) { |
|
62 |
// this.addAuthorsToReference = addAuthorsToReference; |
|
63 |
// } |
|
64 |
} |
Also available in: Unified diff
ref #9145 add IpniSourcesImport for E+M IPNI data