Revision 31938ede
Added by Andreas Müller over 5 years ago
| app-import/src/main/java/eu/etaxonomy/cdm/app/berlinModelImport/EuroMedActivator.java | ||
|---|---|---|
| 66 | 66 |
|
| 67 | 67 |
/** |
| 68 | 68 |
* TODO add the following to a wiki page: |
| 69 |
* HINT: If you are about to import into a mysql data base running under windows and if you wish to dump and restore the resulting data bas under another operation systen |
|
| 69 |
* HINT: If you are about to import into a mysql data base running under windows and if you wish to |
|
| 70 |
* dump and restore the resulting data bas under another operation systen |
|
| 70 | 71 |
* you must set the mysql system variable lower_case_table_names = 0 in order to create data base with table compatible names. |
| 71 | 72 |
* |
| 72 |
* |
|
| 73 | 73 |
* @author a.mueller |
| 74 |
* |
|
| 75 | 74 |
*/ |
| 76 | 75 |
public class EuroMedActivator {
|
| 77 | 76 |
private static final Logger logger = Logger.getLogger(EuroMedActivator.class); |
| 78 | 77 |
|
| 79 | 78 |
//database validation status (create, update, validate ...) |
| 80 |
static DbSchemaValidation hbm2dll = DbSchemaValidation.VALIDATE;
|
|
| 79 |
static DbSchemaValidation hbm2dll = DbSchemaValidation.CREATE;
|
|
| 81 | 80 |
// static final Source berlinModelSource = BerlinModelSources.euroMed_Pub2(); |
| 82 | 81 |
static final Source berlinModelSource = BerlinModelSources.euroMed_BGBM42(); |
| 83 | 82 |
// static final Source berlinModelSource = BerlinModelSources.euroMed_PESI3(); |
| 84 | 83 |
// |
| 85 |
// static final ICdmDataSource cdmDestination = CdmDestinations.localH2();
|
|
| 84 |
static final ICdmDataSource cdmDestination = CdmDestinations.localH2(); |
|
| 86 | 85 |
// static final ICdmDataSource cdmDestination = CdmDestinations.cdm_local_euromed(); |
| 87 |
static final ICdmDataSource cdmDestination = CdmDestinations.cdm_local_euromed();
|
|
| 86 |
// static final ICdmDataSource cdmDestination = CdmDestinations.cdm_local_euromed2();
|
|
| 88 | 87 |
// static final ICdmDataSource cdmDestination = CdmDestinations.cdm_test_euroMed(); |
| 89 | 88 |
|
| 90 | 89 |
//check - import |
| 91 |
static final CHECK check = CHECK.IMPORT_WITHOUT_CHECK;
|
|
| 90 |
static final CHECK check = CHECK.CHECK_ONLY;
|
|
| 92 | 91 |
|
| 93 | 92 |
static final boolean doUser = true; |
| 94 | 93 |
// //authors |
| ... | ... | |
| 100 | 99 |
static final boolean doRelNames = true; |
| 101 | 100 |
static final boolean doNameStatus = true; |
| 102 | 101 |
static final boolean doNameFacts = true; |
| 103 |
//serious types do not exist in E+M except for name types which are handled in name relations |
|
| 104 |
static final boolean doTypes = false; //serious types do not exist in E+M except for name types which are handled in name relations |
|
| 105 | 102 |
|
| 106 | 103 |
//taxa |
| 107 | 104 |
static final boolean doTaxa = true; |
| 108 | 105 |
static final boolean doFacts = true; |
| 109 |
static final boolean doOccurences = false;
|
|
| 106 |
static final boolean doOccurences = true;
|
|
| 110 | 107 |
static final boolean doRelTaxa = true; |
| 111 |
static final boolean doCommonNames = false; //currently takes very long |
|
| 108 |
static final boolean doCommonNames = true; //currently takes very long |
|
| 109 |
|
|
| 110 |
//serious types do not exist in E+M except for name types which are handled in name relations |
|
| 111 |
static final boolean doTypes = false; //serious types do not exist in E+M except for name types which are handled in name relations |
|
| 112 |
|
|
| 112 | 113 |
|
| 113 | 114 |
static final boolean doRunTransmissionEngine = false; // (hbm2dll == DbSchemaValidation.VALIDATE); |
| 114 | 115 |
|
| ... | ... | |
| 308 | 309 |
|
| 309 | 310 |
//create feature tree |
| 310 | 311 |
private void createFeatureTree(BerlinModelImportConfigurator config, |
| 311 |
CdmDefaultImport<BerlinModelImportConfigurator> bmImport) |
|
| 312 |
{
|
|
| 312 |
CdmDefaultImport<BerlinModelImportConfigurator> bmImport){
|
|
| 313 | 313 |
if (config.isDoFacts() && (config.getCheck().isImport() ) ){
|
| 314 | 314 |
try {
|
| 315 | 315 |
ICdmRepository app = bmImport.getCdmAppController(); |
| app-import/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelCommonNamesImport.java | ||
|---|---|---|
| 173 | 173 |
|
| 174 | 174 |
|
| 175 | 175 |
@Override |
| 176 |
public boolean doPartition(ResultSetPartitioner partitioner, BerlinModelImportState state) {
|
|
| 176 |
public boolean doPartition(@SuppressWarnings("rawtypes") ResultSetPartitioner partitioner, BerlinModelImportState state) {
|
|
| 177 | 177 |
boolean success = true ; |
| 178 | 178 |
|
| 179 |
Set<TaxonBase> taxaToSave = new HashSet<>(); |
|
| 179 |
@SuppressWarnings("rawtypes")
|
|
| 180 |
Set<TaxonBase> taxaToSave = new HashSet<>(); |
|
| 180 | 181 |
@SuppressWarnings("unchecked")
|
| 181 | 182 |
Map<String, Taxon> taxonMap = partitioner.getObjectMap(BerlinModelTaxonImport.NAMESPACE); |
| 182 | 183 |
@SuppressWarnings("unchecked")
|
| app-import/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelNameFactsImport.java | ||
|---|---|---|
| 54 | 54 |
/** |
| 55 | 55 |
* @author a.mueller |
| 56 | 56 |
* @since 20.03.2008 |
| 57 |
* @version 1.0 |
|
| 58 | 57 |
*/ |
| 59 | 58 |
@Component |
| 60 | 59 |
public class BerlinModelNameFactsImport extends BerlinModelImportBase {
|
| 61 |
private static final Logger logger = Logger.getLogger(BerlinModelNameFactsImport.class); |
|
| 60 |
private static final long serialVersionUID = 4174085686074314138L; |
|
| 61 |
|
|
| 62 |
private static final Logger logger = Logger.getLogger(BerlinModelNameFactsImport.class); |
|
| 62 | 63 |
|
| 63 | 64 |
public static final String NAMESPACE = "NameFact"; |
| 64 | 65 |
|
| ... | ... | |
| 75 | 76 |
} |
| 76 | 77 |
|
| 77 | 78 |
|
| 78 |
|
|
| 79 |
|
|
| 80 | 79 |
@Override |
| 81 | 80 |
protected String getIdQuery(BerlinModelImportState state) {
|
| 82 | 81 |
if (StringUtils.isNotEmpty(state.getConfig().getNameIdTable())){
|
| ... | ... | |
| 88 | 87 |
} |
| 89 | 88 |
} |
| 90 | 89 |
|
| 91 |
|
|
| 92 |
|
|
| 93 |
|
|
| 94 |
/* (non-Javadoc) |
|
| 95 |
* @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getRecordQuery(eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator) |
|
| 96 |
*/ |
|
| 97 | 90 |
@Override |
| 98 | 91 |
protected String getRecordQuery(BerlinModelImportConfigurator config) {
|
| 99 | 92 |
String strQuery = |
| ... | ... | |
| 106 | 99 |
} |
| 107 | 100 |
|
| 108 | 101 |
@Override |
| 109 |
public boolean doPartition(ResultSetPartitioner partitioner, BerlinModelImportState state) {
|
|
| 102 |
public boolean doPartition(@SuppressWarnings("rawtypes") ResultSetPartitioner partitioner, BerlinModelImportState state) {
|
|
| 110 | 103 |
boolean success = true ; |
| 111 | 104 |
BerlinModelImportConfigurator config = state.getConfig(); |
| 112 | 105 |
Set<TaxonName> nameToSave = new HashSet<>(); |
| ... | ... | |
| 173 | 166 |
taxonNameBase.addDescription(description); |
| 174 | 167 |
} |
| 175 | 168 |
}else if (category.equalsIgnoreCase(NAME_FACT_BIBLIOGRAPHY)){
|
| 176 |
if (StringUtils.isNotBlank(nameFact)){
|
|
| 169 |
if (isNotBlank(nameFact)){
|
|
| 177 | 170 |
TaxonNameDescription description = TaxonNameDescription.NewInstance(); |
| 178 | 171 |
TextData bibliography = TextData.NewInstance(Feature.CITATION()); |
| 179 | 172 |
//TODO language |
| app-import/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelOccurrenceSourceImport.java | ||
|---|---|---|
| 53 | 53 |
|
| 54 | 54 |
|
| 55 | 55 |
private Map<String, Integer> sourceNumberRefIdMap; |
| 56 |
private Set<String> unfoundReferences = new HashSet<String>();
|
|
| 56 |
private Set<String> unfoundReferences = new HashSet<>(); |
|
| 57 | 57 |
|
| 58 | 58 |
|
| 59 | 59 |
public BerlinModelOccurrenceSourceImport(){
|
| ... | ... | |
| 83 | 83 |
|
| 84 | 84 |
@Override |
| 85 | 85 |
protected void doInvoke(BerlinModelImportState state) {
|
| 86 |
unfoundReferences = new HashSet<String>();
|
|
| 86 |
unfoundReferences = new HashSet<>(); |
|
| 87 | 87 |
|
| 88 | 88 |
try {
|
| 89 | 89 |
sourceNumberRefIdMap = makeSourceNumberReferenceIdMap(state); |
| ... | ... | |
| 101 | 101 |
} |
| 102 | 102 |
|
| 103 | 103 |
@Override |
| 104 |
public boolean doPartition(ResultSetPartitioner partitioner, BerlinModelImportState state) {
|
|
| 104 |
public boolean doPartition(@SuppressWarnings("rawtypes") ResultSetPartitioner partitioner, BerlinModelImportState state) {
|
|
| 105 | 105 |
boolean success = true; |
| 106 | 106 |
ResultSet rs = partitioner.getResultSet(); |
| 107 |
Map<String, Reference> refMap = partitioner.getObjectMap(BerlinModelReferenceImport.REFERENCE_NAMESPACE); |
|
| 107 |
@SuppressWarnings("unchecked")
|
|
| 108 |
Map<String, Reference> refMap = partitioner.getObjectMap(BerlinModelReferenceImport.REFERENCE_NAMESPACE); |
|
| 108 | 109 |
|
| 109 | 110 |
Set<DescriptionElementBase> objectsToSave = new HashSet<DescriptionElementBase>(); |
| 110 | 111 |
try {
|
| ... | ... | |
| 165 | 166 |
String nameSpace; |
| 166 | 167 |
Class<?> cdmClass; |
| 167 | 168 |
Set<String> idSet; |
| 168 |
Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<Object, Map<String, ? extends CdmBase>>();
|
|
| 169 |
Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<>(); |
|
| 169 | 170 |
|
| 170 | 171 |
try{
|
| 171 |
Set<String> occurrenceIdSet = new HashSet<String>();
|
|
| 172 |
Set<String> referenceIdSet = new HashSet<String>();
|
|
| 173 |
Set<String> nameIdSet = new HashSet<String>();
|
|
| 174 |
Set<String> sourceNumberSet = new HashSet<String>();
|
|
| 172 |
Set<String> occurrenceIdSet = new HashSet<>(); |
|
| 173 |
Set<String> referenceIdSet = new HashSet<>(); |
|
| 174 |
Set<String> nameIdSet = new HashSet<>(); |
|
| 175 |
Set<String> sourceNumberSet = new HashSet<>(); |
|
| 175 | 176 |
while (rs.next()){
|
| 176 | 177 |
handleForeignKey(rs, occurrenceIdSet, "occurrenceFk"); |
| 177 | 178 |
handleForeignKey(rs, nameIdSet, "oldNameFk"); |
| ... | ... | |
| 179 | 180 |
} |
| 180 | 181 |
|
| 181 | 182 |
sourceNumberSet.remove("");
|
| 182 |
referenceIdSet = handleSourceNumber(rs, sourceNumberSet, result);
|
|
| 183 |
referenceIdSet = handleSourceNumber(sourceNumberSet);
|
|
| 183 | 184 |
|
| 184 | 185 |
|
| 185 | 186 |
//occurrence map |
| 186 | 187 |
nameSpace = BerlinModelOccurrenceImport.NAMESPACE; |
| 187 | 188 |
cdmClass = Distribution.class; |
| 188 | 189 |
idSet = occurrenceIdSet; |
| 189 |
Map<String, Distribution> occurrenceMap = (Map<String, Distribution>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace); |
|
| 190 |
@SuppressWarnings("unchecked")
|
|
| 191 |
Map<String, Distribution> occurrenceMap = (Map<String, Distribution>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace); |
|
| 190 | 192 |
result.put(nameSpace, occurrenceMap); |
| 191 | 193 |
|
| 192 | 194 |
//name map |
| 193 | 195 |
nameSpace = BerlinModelTaxonNameImport.NAMESPACE; |
| 194 | 196 |
cdmClass = TaxonName.class; |
| 195 | 197 |
idSet =nameIdSet; |
| 196 |
Map<String, TaxonName> nameMap = (Map<String, TaxonName>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace); |
|
| 198 |
@SuppressWarnings("unchecked")
|
|
| 199 |
Map<String, TaxonName> nameMap = (Map<String, TaxonName>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace); |
|
| 197 | 200 |
result.put(nameSpace, nameMap); |
| 198 | 201 |
|
| 199 | 202 |
//reference map |
| 200 | 203 |
nameSpace = BerlinModelReferenceImport.REFERENCE_NAMESPACE; |
| 201 | 204 |
cdmClass = Reference.class; |
| 202 | 205 |
idSet = referenceIdSet; |
| 203 |
Map<String, Reference> referenceMap = (Map<String, Reference>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace); |
|
| 206 |
@SuppressWarnings("unchecked")
|
|
| 207 |
Map<String, Reference> referenceMap = (Map<String, Reference>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace); |
|
| 204 | 208 |
result.put(nameSpace, referenceMap); |
| 205 | 209 |
|
| 206 | 210 |
} catch (SQLException e) {
|
| ... | ... | |
| 209 | 213 |
return result; |
| 210 | 214 |
} |
| 211 | 215 |
|
| 212 |
private Set<String> handleSourceNumber(ResultSet rs, Set<String> sourceNumberSet, Map<Object, Map<String, ? extends CdmBase>> result) {
|
|
| 216 |
private Set<String> handleSourceNumber(Set<String> sourceNumberSet) {
|
|
| 213 | 217 |
Map<String, Integer> sourceNumberReferenceIdMap = this.sourceNumberRefIdMap; |
| 214 |
Set<String> referenceIdSet = new HashSet<String>();
|
|
| 218 |
Set<String> referenceIdSet = new HashSet<>(); |
|
| 215 | 219 |
|
| 216 | 220 |
for(String sourceNumber : sourceNumberSet){
|
| 217 | 221 |
Integer refId = sourceNumberReferenceIdMap.get(sourceNumber); |
| ... | ... | |
| 258 | 262 |
* @throws SQLException |
| 259 | 263 |
*/ |
| 260 | 264 |
private Map<String, Integer> makeSourceNumberReferenceIdMap(BerlinModelImportState state) throws SQLException {
|
| 261 |
Map<String, Integer> result = new HashMap<String, Integer>();
|
|
| 265 |
Map<String, Integer> result = new HashMap<>(); |
|
| 262 | 266 |
|
| 263 | 267 |
Source source = state.getConfig().getSource(); |
| 264 | 268 |
String strQuery = " SELECT RefId, IdInSource " + |
| 265 | 269 |
" FROM Reference " + |
| 266 |
" WHERE (IdInSource IS NOT NULL) AND (IdInSource NOT LIKE '') ";
|
|
| 270 |
" WHERE (IdInSource IS NOT NULL) AND (IdInSource NOT LIKE '') "; |
|
| 267 | 271 |
|
| 268 | 272 |
ResultSet rs = source.getResultSet(strQuery) ; |
| 269 | 273 |
while (rs.next()){
|
Also available in: Unified diff
cleanup