Project

General

Profile

Revision 5cf1f8e1

ID5cf1f8e11764921820a2054f583fe2add2223080
Parent 49db4e78
Child fd88fb6a

Added by Andreas Müller 5 months ago

ref #9061 some further steps for an import analyze framework

View differences:

cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/common/CdmApplicationAwareDefaultImport.java
9 9

  
10 10
package eu.etaxonomy.cdm.io.common;
11 11

  
12
import java.lang.reflect.Constructor;
13
import java.lang.reflect.InvocationTargetException;
12 14
import java.util.HashMap;
13 15
import java.util.Map;
14 16

  
......
23 25
import org.springframework.security.core.context.SecurityContextHolder;
24 26
import org.springframework.stereotype.Component;
25 27

  
28
import eu.etaxonomy.cdm.api.application.CdmRepository;
26 29
import eu.etaxonomy.cdm.api.service.IService;
27 30
import eu.etaxonomy.cdm.io.common.events.IIoObserver;
31
import eu.etaxonomy.cdm.io.fact.altitude.in.analyze.ExcelFormatAnalyzeResult;
32
import eu.etaxonomy.cdm.io.fact.altitude.in.analyze.ExcelFormatAnalyzer;
33
import eu.etaxonomy.cdm.io.fact.in.FactExcelImportConfiguratorBase;
28 34
import eu.etaxonomy.cdm.model.common.CdmBase;
29 35

  
30 36
/**
......
110 116
        state.initialize(config);
111 117
        state.setCheck(true);
112 118

  
113
        //do check for each class
114
        for (Class<ICdmImport> ioClass: config.getIoClassList()){
119
        if (config instanceof FactExcelImportConfiguratorBase<?>){
120
            ExcelFormatAnalyzer<?> analyzer = ((FactExcelImportConfiguratorBase<?>)config).getAnalyzer();
121
            ExcelFormatAnalyzeResult analyzeResult = analyzer.invoke();
122
            //TODO very preliminary for development only
123
            System.out.println(analyzeResult.toString());
115 124
            try {
116
                String ioBeanName = getComponentBeanName(ioClass);
117
                ICdmIO cdmIo = applicationContext.getBean(ioBeanName, ICdmIO.class);
118
                if (cdmIo != null){
119
                    registerObservers(config, cdmIo);
120
                    state.setCurrentIO(cdmIo);
121
                    result &= cdmIo.check(state);
122
                    unRegisterObservers(config, cdmIo);
123
                }else{
124
                    logger.error("cdmIO was null");
125
                    result = false;
125
                for (Class<ICdmImport> ioClass: config.getIoClassList()){
126
                    Constructor<ICdmImport> constructor = ioClass.getConstructor();
127
                    constructor.setAccessible(true);
128
                    ICdmImport ioInstance = constructor.newInstance();
129
                    CdmRepository repository = (CdmRepository)applicationContext.getBean("CdmRepository");
130
                    ioInstance.setRepository(repository);
131
//                    ioInstance.analyze();
126 132
                }
127
            } catch (Exception e) {
133
            } catch (NoSuchMethodException | SecurityException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
134
                analyzeResult.addFatalError("Problem accessing the constructor of the import class. Please contact your system developers");
135
            }
136
            if (analyzeResult.hasErrors()){
137
                result = false;
138
            }
139
        }else{
140
            //do check for each class
141
            for (Class<ICdmImport> ioClass: config.getIoClassList()){
142
                try {
143
                    String ioBeanName = getComponentBeanName(ioClass);
144
                    ICdmIO cdmIo = applicationContext.getBean(ioBeanName, ICdmIO.class);
145
                    if (cdmIo != null){
146
                        registerObservers(config, cdmIo);
147
                        state.setCurrentIO(cdmIo);
148
                        result &= cdmIo.check(state);
149
                        unRegisterObservers(config, cdmIo);
150
                    }else{
151
                        logger.error("cdmIO was null");
152
                        result &= false;
153
                    }
154
                } catch (Exception e) {
128 155
                    logger.error(e);
129 156
                    e.printStackTrace();
130
                    result = false;
157
                    result &= false;
158
                }
131 159
            }
132 160
        }
133 161

  
162

  
134 163
        //return
135 164
        System.out.println("End checking Source ("+ config.getSourceNameString() + ") for import to Cdm");
136 165
        state.setCheck(false);
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/common/CdmImportBase.java
22 22

  
23 23
import org.apache.log4j.Logger;
24 24

  
25
import eu.etaxonomy.cdm.api.application.ICdmRepository;
25 26
import eu.etaxonomy.cdm.api.service.pager.Pager;
26 27
import eu.etaxonomy.cdm.common.media.ImageInfo;
27 28
import eu.etaxonomy.cdm.io.common.mapping.IInputTransformer;
......
156 157
 		}
157 158
	}
158 159

  
160
    protected ICdmRepository repository;
159 161

  
160
	}
162
    @Override
163
    public void setRepository(ICdmRepository repository){
164
        this.repository = repository;
165
    }
161 166

  
162 167
    @Override
163 168
    protected ImportResult getNoDataResult(STATE state) {
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/common/ICdmImport.java
6 6

  
7 7
package eu.etaxonomy.cdm.io.common;
8 8

  
9
import eu.etaxonomy.cdm.api.application.ICdmRepository;
9 10

  
10 11
/**
11 12
 * Interface for all import classes.
......
21 22

  
22 23
    public abstract void invoke(STATE state);
23 24

  
25
    public void setRepository(ICdmRepository repository);
26

  
24 27
}
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/fact/altitude/in/AltitudeExcelFormatAnalyzer.java
14 14
 * @author a.mueller
15 15
 * @since 02.06.2020
16 16
 */
17
public class AltitudeExcelFormatAnalyzer extends ExcelFormatAnalyzer {
17
public class AltitudeExcelFormatAnalyzer
18
            extends ExcelFormatAnalyzer<AltitudeExcelImportConfigurator> {
18 19

  
19
    private AltitudeExcelFormatAnalyzer() {
20
        super(requiredColumns(), optionalColumns(), optionalMultiColumns());
21
        // TODO Auto-generated constructor stub
20
    protected AltitudeExcelFormatAnalyzer(AltitudeExcelImportConfigurator config) {
21
        super(config, requiredWorksheets(), requiredColumns(), optionalColumns(), optionalMultiColumns());
22
    }
23

  
24
    private static String[] requiredWorksheets() {
25
        return new String[]{"Altitude"};
22 26
    }
23 27

  
24 28
    private static String[] requiredColumns() {
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/fact/altitude/in/AltitudeExcelImportConfigurator.java
21 21
 * @since 28.05.2020
22 22
 */
23 23
public class AltitudeExcelImportConfigurator
24
        extends FactExcelImportConfiguratorBase{
24
        extends FactExcelImportConfiguratorBase<AltitudeExcelFormatAnalyzer>{
25 25

  
26 26
    private static final long serialVersionUID = -6403743396163163359L;
27 27

  
......
60 60
        };
61 61
    }
62 62

  
63
    @Override
64
    public AltitudeExcelFormatAnalyzer getAnalyzer() {
65
        return new AltitudeExcelFormatAnalyzer(this);
66
    }
67

  
63 68
    public UUID getFeatureUuid() {
64 69
        return featureUuid;
65 70
    }
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/fact/altitude/in/analyze/AnalyzeResultSeverity.java
1
/**
2
* Copyright (C) 2020 EDIT
3
* European Distributed Institute of Taxonomy
4
* http://www.e-taxonomy.eu
5
*
6
* The contents of this file are subject to the Mozilla Public License Version 1.1
7
* See LICENSE.TXT at the top of this package for the full license terms.
8
*/
9
package eu.etaxonomy.cdm.io.fact.altitude.in.analyze;
10

  
11
/**
12
 * @author a.mueller
13
 * @since 01.07.2020
14
 */
15
public enum AnalyzeResultSeverity {
16
    FATAL,
17
    ERROR,
18
    WARNING,
19
    INFO;
20
}
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/fact/altitude/in/analyze/ExcelFormatAnalyzeResult.java
12 12
import java.util.List;
13 13

  
14 14
/**
15
 * Result for an {@link ExcelFormatAnalyzer}
16
 *
15 17
 * @author a.mueller
16 18
 * @since 02.06.2020
17 19
 */
18 20
public class ExcelFormatAnalyzeResult {
19 21

  
22
    private List<FormatAnalyzeInfo> fatalErrors = new ArrayList<>();
20 23
    private List<FormatAnalyzeInfo> errors = new ArrayList<>();
21 24
    private List<FormatAnalyzeInfo> warnings = new ArrayList<>();
22 25
    private List<FormatAnalyzeInfo> infos = new ArrayList<>();
23 26

  
24
    public void addError(FormatAnalyzeInfo error) {
27
    private ExcelFormatAnalyzer<?> analyzer;
28

  
29
    public ExcelFormatAnalyzeResult(ExcelFormatAnalyzer<?> excelFormatAnalyzer) {
30
        this.analyzer = excelFormatAnalyzer;
31
    }
32

  
33
    public void addFatalError(String description) {
34
        FormatAnalyzeInfo error = FormatAnalyzeInfo.NewInstance(AnalyzeResultSeverity.FATAL, description);
35
        fatalErrors.add(error);
36
    }
37

  
38
    public void addError(String description) {
39
        FormatAnalyzeInfo error = FormatAnalyzeInfo.NewInstance(AnalyzeResultSeverity.ERROR, description);
25 40
        errors.add(error);
26 41
    }
27 42

  
28
    public void addWarning(FormatAnalyzeInfo warning) {
43
    public void addWarning(String description) {
44
        FormatAnalyzeInfo warning = FormatAnalyzeInfo.NewInstance(AnalyzeResultSeverity.ERROR, description);
29 45
        warnings.add(warning);
30 46
    }
31 47

  
32
    public void addInfo(FormatAnalyzeInfo info) {
48
    public void addInfo(String description) {
49
        FormatAnalyzeInfo info = FormatAnalyzeInfo.NewInstance(AnalyzeResultSeverity.ERROR, description);
33 50
        infos.add(info);
34 51
    }
52

  
53
    public boolean hasFatalErrors() {
54
        return !fatalErrors.isEmpty();
55
    }
56

  
57
    public boolean hasErrors() {
58
        return hasFatalErrors() || !errors.isEmpty() ;
59
    }
60

  
61
    @Override
62
    public String toString() {
63
        StringBuilder result = new StringBuilder(underline("\nAnalyze result"));
64
        //fatal
65
        int i = 1;
66
        result.append(hasFatalErrors()? underline("Fatal errors"): "No fatal errors.\n");
67
        for (FormatAnalyzeInfo info : fatalErrors){
68
            result.append(i + ": " + info.toString());
69
        }
70
        //error
71
        result.append(!errors.isEmpty()? underline("Errors"): "No errors.\n");
72
        for (FormatAnalyzeInfo info : errors){
73
            result.append(i + ": " + info.toString());
74
        }
75
        //warning
76
        result.append(!warnings.isEmpty()? underline("Warnings"): "No warnings.\n");
77
        for (FormatAnalyzeInfo info : errors){
78
            result.append(i + ": " + info.toString());
79
        }
80
        //info
81
        result.append(!infos.isEmpty()? underline("Infos"): "No infos.\n");
82
        for (FormatAnalyzeInfo info : infos){
83
            result.append(i + ": " + info.toString());
84
        }
85
        result.append("\nEnd analyze result.\n");
86

  
87
        return result.toString();
88
    }
89

  
90
    private StringBuilder underline(String string) {
91
        StringBuilder result = new StringBuilder(string.length()*2+3);
92
        result.append(string);
93
        result.append(":\n");
94
        for (@SuppressWarnings("unused") char c : string.toCharArray()){
95
            result.append("=");
96
        }
97
        result.append("\n");
98
        return result;
99
    }
35 100
}
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/fact/altitude/in/analyze/ExcelFormatAnalyzer.java
8 8
*/
9 9
package eu.etaxonomy.cdm.io.fact.altitude.in.analyze;
10 10

  
11
import java.io.FileNotFoundException;
12
import java.io.InputStream;
13
import java.net.URI;
11 14
import java.util.ArrayList;
12 15
import java.util.Arrays;
13 16
import java.util.List;
14 17

  
18
import org.apache.poi.EncryptedDocumentException;
19
import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
20
import org.apache.poi.ss.usermodel.Sheet;
21
import org.apache.poi.ss.usermodel.Workbook;
22
import org.apache.poi.ss.usermodel.WorkbookFactory;
23

  
24
import eu.etaxonomy.cdm.common.UriUtils;
25
import eu.etaxonomy.cdm.io.fact.in.FactExcelImportConfiguratorBase;
26

  
15 27
/**
16 28
 * @author a.mueller
17 29
 * @since 02.06.2020
18 30
 */
19
public class ExcelFormatAnalyzer {
31
public class ExcelFormatAnalyzer<CONFIG extends FactExcelImportConfiguratorBase<?>> {
32

  
33
    private CONFIG config;
34

  
35
    private List<String> requiredWorksheets = new ArrayList<>();
20 36

  
21 37
    private List<String> requiredColumns = new ArrayList<>();
22 38

  
......
24 40

  
25 41
    private List<String> optionalMultiColumns = new ArrayList<>();
26 42

  
27
    public ExcelFormatAnalyzer(String[] requiredColumns, String[] optionalColumns,
43
    public ExcelFormatAnalyzer(CONFIG config,
44
            String[] requiredWorksheets,
45
            String[] requiredColumns,
46
            String[] optionalColumns,
28 47
            String[] optionalMultiColumns) {
48
        this.config = config;
49
        this.requiredWorksheets.addAll(Arrays.asList(requiredWorksheets));
29 50
        this.requiredColumns.addAll(Arrays.asList(requiredColumns));
30 51
        this.optionalColumns.addAll(Arrays.asList(optionalColumns));
31 52
        this.optionalMultiColumns.addAll(Arrays.asList(optionalMultiColumns));
......
57 78
//************************* METHOD ************************/
58 79

  
59 80
    public ExcelFormatAnalyzeResult invoke(){
60
        ExcelFormatAnalyzeResult result = new ExcelFormatAnalyzeResult();
81
        ExcelFormatAnalyzeResult result = new ExcelFormatAnalyzeResult(this);
61 82

  
62
        analyzeWorkbook(result);
83
        //workbook format
84
        analyzeWorkbookFormat(result);
85
        if (result.hasFatalErrors()){
86
            return result;
87
        }
63 88

  
64
        result.addError(FormatAnalyzeInfo.NewInstance("Obligatory column xxx is missing. Running import not possible"));
89
//        result.addError("Obligatory column xxx is missing. Running import not possible");
65 90
        return result;
66 91
    }
67 92

  
68
    private void analyzeWorkbook(ExcelFormatAnalyzeResult result) {
69
        // TODO Auto-generated method stub
93
    protected void analyzeWorkbookFormat(ExcelFormatAnalyzeResult result) {
94
        URI uri = config.getSource();
95
        if (uri == null){
96
            result.addFatalError("Now source defined. Import not possible.");
97
            return;
98
        }
99
        try {
100
            InputStream stream = UriUtils.getInputStream(uri);
101
            Workbook wb = WorkbookFactory.create(stream);
102

  
103
            List<String> worksheetNames = new ArrayList<>();
104
            worksheetNames.add(config.getWorksheetName());
105
            for (String worksheetName : worksheetNames){
106
                analyzeWorksheetName(result, wb, worksheetNames, worksheetName);
107
            }
108
        } catch(FileNotFoundException fne) {
109
            result.addFatalError("Import file '" + uri.toString() + "' not found. Import not possible.");
110
        } catch(InvalidFormatException ife) {
111
            result.addFatalError("Import file has invalid format for an excel file. Import not possible.");
112
        } catch(EncryptedDocumentException ede) {
113
            result.addFatalError("File is encrypted. Import not possible.");
114
        } catch(Exception ioe) {
115
            result.addFatalError("Unhandled exception  when reading '" + uri.toString() + "'. Import not possible.");
116
        }
117
    }
70 118

  
119
    private void analyzeWorksheetName(ExcelFormatAnalyzeResult result, Workbook wb, List<String> worksheetNames,
120
            String worksheetName) {
121
        try {
122
            Sheet worksheet = wb.getSheet(worksheetName);
123
            if(worksheet == null && worksheetNames.size() == 1){
124
                worksheet = wb.getSheetAt(0);
125
            }
126
            if (worksheet == null){
127
                result.addFatalError("Required worksheet "+worksheetName+" not found in file. Import not possible.");
128
            }
129
        } catch (Exception e) {
130
            result.addFatalError("Error when reading worksheet '" + worksheetName + "' not found. Import not possible.");
131
        }
71 132
    }
72 133
}
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/fact/altitude/in/analyze/FormatAnalyzeInfo.java
14 14
 */
15 15
public class FormatAnalyzeInfo {
16 16

  
17
    private AnalyzeResultSeverity severity;
17 18
    private String description;
18 19

  
19
    public static FormatAnalyzeInfo NewInstance(String description) {
20
        FormatAnalyzeInfo result = new FormatAnalyzeInfo();
21
        result.description = description;
20
    public static FormatAnalyzeInfo NewInstance(AnalyzeResultSeverity severity, String description) {
21
        FormatAnalyzeInfo result = new FormatAnalyzeInfo(severity, description);
22 22
        return result;
23 23
    }
24 24

  
25
// ************************* GETTER /SETTER **********************/
26

  
27
    private FormatAnalyzeInfo(AnalyzeResultSeverity severity, String description) {
28
        this.setSeverity(severity);
29
        this.description = description;
30
    }
31

  
25 32
    public String getDescription() {
26 33
        return description;
27 34
    }
......
29 36
        this.description = description;
30 37
    }
31 38

  
39
    public AnalyzeResultSeverity getSeverity() {
40
        return severity;
41
    }
42
    public void setSeverity(AnalyzeResultSeverity severity) {
43
        this.severity = severity;
44
    }
32 45

  
33

  
46
    @Override
47
    public String toString() {
48
        return severity.toString() + ":" + getDescription();
49
    }
34 50
}
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/fact/in/FactExcelImportConfiguratorBase.java
13 13
import eu.etaxonomy.cdm.database.ICdmDataSource;
14 14
import eu.etaxonomy.cdm.io.common.mapping.IInputTransformer;
15 15
import eu.etaxonomy.cdm.io.excel.common.ExcelImportConfiguratorBase;
16
import eu.etaxonomy.cdm.io.fact.altitude.in.analyze.ExcelFormatAnalyzer;
16 17

  
17 18
/**
18 19
 * Configurator base class for taxon fact excel imports.
......
20 21
 * @author a.mueller
21 22
 * @since 28.05.2020
22 23
 */
23
public abstract class FactExcelImportConfiguratorBase
24
public abstract class FactExcelImportConfiguratorBase<A extends ExcelFormatAnalyzer<?>>
24 25
        extends ExcelImportConfiguratorBase{
25 26

  
26 27
    private static final long serialVersionUID = 1649010514975388511L;
......
29 30
        super(uri, destination, transformer);
30 31
    }
31 32

  
33
    public abstract A getAnalyzer();
34

  
32 35
}

Also available in: Unified diff

Add picture from clipboard (Maximum size: 40 MB)