surveyer.surveyreader

  1"""Read info from survey export and make useable"""
  2
  3import pandas as pd
  4
  5class DataSet:
  6    def __init__(self, filepath):
  7        self.metadata = {}
  8        self.data = pd.DataFrame()
  9
 10        self.read(filepath)
 11        self.preprocess()
 12
 13    def read(self, filepath):
 14        """Reading the survey results from the export from EU survey"""
 15
 16        if filepath.find("xls") > -1:
 17            self.data = pd.read_excel(filepath, header=3)
 18        elif filepath.find("h5") > -1:
 19            self.data = pd.read_hdf(filepath, key="survey")
 20
 21    def _anonymize(self, internaluse=False, outpath=""):
 22        """Deleting entries with personal information.
 23        If internaluse = True, keeping details on affiliation.
 24        If outpath is provided, anonymized dataset is written to xls."""
 25
 26        if "Name" in self.data.columns:
 27            self.data = self.data.drop(
 28                columns=[
 29                    "Name",
 30                    "email",
 31                    "Invitation number",
 32                    "Contribution ID",
 33                    "User name",
 34                    "Languages",
 35                ]
 36            )
 37
 38        if (
 39            not internaluse
 40            and "Further description of your role or group" in self.data.columns
 41        ):
 42            self.data = self.data.drop(
 43                columns=[
 44                    "Your institution / organisation / group (for internal purposes only, information will not be public).",
 45                    "Further description of your role or group",
 46                ]
 47            )
 48
 49        if outpath:
 50            self.data.to_hdf(outpath, key="survey")
 51
 52    def _adapt_column_names(self):
 53        """Adapt entries and columns of the table"""
 54
 55        for colname in self.data.columns:
 56            colname_new = str(colname).replace("applicable:", "applicable ")
 57            colname_new = str(colname).replace(": Confidence", "- Confidence")
 58            self.data = self.data.rename(columns={colname: colname_new})
 59
 60    def _create_question_metadata(self):
 61        """Extracting metadata from the survey file.
 62        For each question, an identifier (A...) is defined, and subquestions grouped."""
 63        questions = {}
 64
 65        for colname in self.data.columns:
 66            subtexts = colname.split(":")
 67            questionid = False
 68            for qid in questions:
 69                if questions[qid]["question"] == subtexts[0]:
 70                    questionid = qid
 71            if not questionid:
 72                if len(subtexts) == 1:
 73                    subtexts[0] = colname
 74                questions.setdefault(
 75                    "A" + str(len(questions)),
 76                    {
 77                        "question": subtexts[0],
 78                        "subquestions": [],
 79                        "colnames": [],
 80                        "entrytype": "undefined",
 81                        "params": {},
 82                    },
 83                )
 84                qid = "A" + str(len(questions) - 1)
 85
 86            if len(subtexts) > 1:
 87                questions[qid]["subquestions"].append(subtexts[1 : len(subtexts)])
 88                questions[qid]["colnames"].append(colname)
 89        self.metadata = questions
 90
 91    def preprocess(self):
 92        """Preprocessing the survey data for use"""
 93
 94        self._anonymize()
 95        self._adapt_column_names()
 96        self._create_question_metadata()
 97
 98        self._identify_question_types()
 99
100        self._adapt_answers()
101
102    def _identify_question_types(self):
103        """Identifying question types according to metadata"""
104
105        for qid in self.metadata:
106
107            columns = []
108            self.metadata[qid]["params"].setdefault("subtypes", {})
109            self.metadata[qid]["params"].setdefault("options", {})
110
111            if len(self.metadata[qid]["colnames"]) == 0:
112                columns = [self.metadata[qid]["question"]]
113            else:
114                columns = self.metadata[qid]["colnames"]
115                self.metadata[qid]["entrytype"] = "multiple"
116
117            for colname in columns:
118                columntype = ""
119
120                answersall = [x for x in self.data[colname] if pd.notna(x)]
121                uniqueanswers = list(set(answersall))
122
123                dtype = self.data[colname].dtype
124
125                if str(dtype) == "float64":
126                    columntype = "undefined"
127
128                elif str(dtype) == "datetime64[ns]":
129                    columntype = "date"
130
131                elif len(answersall) == 0:
132                    columntype = "no answers"
133
134                elif isinstance(uniqueanswers[0], str):
135                    columntype = "text"
136
137                    subanswers = []
138                    samelength = True
139                    entrylength = 1
140                    notsingle = False
141
142                    for answers in answersall:
143                        answerssplit = answers.split(";")
144                        if entrylength == 1:
145                            entrylength = len(answerssplit)
146
147                        if answers == answersall[0] and len(answerssplit) > 1:
148                            notsingle = True
149
150                        if len(answerssplit) != entrylength and samelength:
151                            samelength = False
152
153                        for subans in answerssplit:
154                            if not subans.lstrip(" ") in subanswers:
155                                subanswers.append(subans.lstrip(" "))
156
157                    # see if answers contain /10 -> rating
158
159                    if (
160                        str(uniqueanswers[0]).find("/5") > 0
161                        or str(uniqueanswers[0]).find("/10") > 0
162                    ) and str(uniqueanswers[0]).find("ttp") < 0:
163                        columntype = "rating"
164
165                    # see if entries can be split with different length ->
166                    # multichoice
167
168                    elif entrylength > 1 and not samelength:
169                        columntype = "enumerate"
170
171                    # see if entries can be split with same length -> ranking
172
173                    elif entrylength > 1 and samelength and notsingle:
174                        columntype = "ranking"
175
176                    # see if entries are repetitive -> select
177
178                    elif entrylength == 1 and len(subanswers) < len(answersall):
179                        columntype = "select"
180
181                    if columntype != "text":
182                        if columntype == "rating":
183                            self.metadata[qid]["params"]["options"].setdefault(
184                                "factor", int(str(uniqueanswers[0]).split("/")[1])
185                            )
186                        else:
187                            self.metadata[qid]["params"]["options"].setdefault(
188                                colname, subanswers
189                            )
190
191                if not columntype:
192                    columntype = "undefined"
193
194                if self.metadata[qid]["entrytype"] == "multiple":
195                    self.metadata[qid]["params"]["subtypes"].setdefault(
196                        colname, columntype
197                    )
198                else:
199                    self.metadata[qid]["entrytype"] = columntype
200
201    def _adapt_answers(self):
202        """Change the entries in the data table for better processing"""
203        pass
204
205    def list_questions(self, filter_type=""):
206        """Returns DataFrame with metadata of the questions.
207        If filter_type provides type of question, only matching entries are listed
208        """
209
210        meta_df = pd.DataFrame(self.metadata).T
211        if filter_type:
212            meta_df = meta_df[meta_df["entrytype"] == filter_type]
213        return meta_df
214
215    def get_question_data(self, questionid):
216        """Returns dataframe according to question identifier"""
217        colnames = []
218        if len(self.metadata[questionid]["colnames"]) > 0:
219            colnames = self.metadata[questionid]["colnames"]
220        else:
221            colnames = [self.metadata[questionid]["question"]]
222
223        return self.data.loc[:, colnames]
224
225    def _get_colnames(self, questionids, acceptedtypes=[]):
226        """Returns column names for given question IDs, facilitating filtering for specific types."""
227
228        colnames = []
229
230        for qid in questionids:
231            if (
232                acceptedtypes
233                and self.metadata[qid]["entrytype"] in acceptedtypes
234                or not acceptedtypes
235            ):
236                colnames.append(self.metadata[qid]["question"])
237            elif self.metadata[qid]["entrytype"] == "multiple":
238                for colname in self.metadata[qid]["params"]["subtypes"]:
239                    if (
240                        self.metadata[qid]["params"]["subtypes"][colname]
241                        in acceptedtypes
242                    ):
243                        colnames.append(colname)
244            else:
245                print(
246                    "For question-ID",
247                    qid,
248                    "the type is not an accepted type ",
249                    acceptedtypes,
250                    " or 'multiple', will not add type",
251                    self.metadata[qid]["entrytype"],
252                )
253
254        return colnames
255
256    def extract_subset(self, questionids, acceptedtypes=[]):
257        """Returns dataframe with subset of data according to questionids, and filters for provided types of questions."""
258        colnames = self._get_colnames(questionids, acceptedtypes)
259
260        return self.data[colnames]