The following are 30 code examples of pandas.errors.EmptyDataError(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module pandas.errors, or try the search function
Example #1
def test_skiprows_callable(self): data = 'a\n1\n2\n3\n4\n5' skiprows = lambda x: x % 2 == 0 expected = DataFrame({'1': [3, 5]}) df = self.read_csv(StringIO(data), skiprows=skiprows) tm.assert_frame_equal(df, expected) expected = DataFrame({'foo': [3, 5]}) df = self.read_csv(StringIO(data), skiprows=skiprows, header=0, names=['foo']) tm.assert_frame_equal(df, expected) skiprows = lambda x: True msg = "No columns to parse from file" with tm.assert_raises_regex(EmptyDataError, msg): self.read_csv(StringIO(data), skiprows=skiprows) # This is a bad callable and should raise. msg = "by zero" skiprows = lambda x: 1 / 0 with tm.assert_raises_regex(ZeroDivisionError, msg): self.read_csv(StringIO(data), skiprows=skiprows)
Example #2
def test_read_empty_with_usecols(self): # See gh-12493 names = ['Dummy', 'X', 'Dummy_2'] usecols = names[1:2] # ['X'] # first, check to see that the response of # parser when faced with no provided columns # throws the correct error, with or without usecols errmsg = "No columns to parse from file" with tm.assert_raises_regex(EmptyDataError, errmsg): self.read_csv(StringIO('')) with tm.assert_raises_regex(EmptyDataError, errmsg): self.read_csv(StringIO(''), usecols=usecols) expected = DataFrame(columns=usecols, index=[0], dtype=np.float64) df = self.read_csv(StringIO(',,'), names=names, usecols=usecols) tm.assert_frame_equal(df, expected) expected = DataFrame(columns=usecols) df = self.read_csv(StringIO(''), names=names, usecols=usecols) tm.assert_frame_equal(df, expected)
Example #3
def _read_csv(self, filename, without_header=False): params = { 'delimiter': ';', 'encoding': 'ISO-8859-1', 'quotechar': '"', # Avoiding converting string to int (nr_cpf_candidato) 'dtype': object } if without_header: params['header'] = None try: df = read_csv(filename, **params) return df except EmptyDataError: logging.info(f'{filename} is empty') return []
Example #4
def detect_colspecs(self, n=100, skiprows=None): # Regex escape the delimiters delimiters = ''.join([r'\%s' % x for x in self.delimiter]) pattern = re.compile('([^%s]+)' % delimiters) rows = self.get_rows(n, skiprows) if not rows: raise EmptyDataError("No rows from which to infer column width") max_len = max(map(len, rows)) mask = np.zeros(max_len + 1, dtype=int) if self.comment is not None: rows = [row.partition(self.comment)[0] for row in rows] for row in rows: for m in pattern.finditer(row): mask[m.start():m.end()] = 1 shifted = np.roll(mask, 1) shifted[0] = 0 edges = np.where((mask ^ shifted) == 1)[0] edge_pairs = list(zip(edges[::2], edges[1::2])) return edge_pairs
Example #5
def test_skiprows_callable(self): data = 'a\n1\n2\n3\n4\n5' skiprows = lambda x: x % 2 == 0 expected = DataFrame({'1': [3, 5]}) df = self.read_csv(StringIO(data), skiprows=skiprows) tm.assert_frame_equal(df, expected) expected = DataFrame({'foo': [3, 5]}) df = self.read_csv(StringIO(data), skiprows=skiprows, header=0, names=['foo']) tm.assert_frame_equal(df, expected) skiprows = lambda x: True msg = "No columns to parse from file" with tm.assert_raises_regex(EmptyDataError, msg): self.read_csv(StringIO(data), skiprows=skiprows) # This is a bad callable and should raise. msg = "by zero" skiprows = lambda x: 1 / 0 with tm.assert_raises_regex(ZeroDivisionError, msg): self.read_csv(StringIO(data), skiprows=skiprows)
Example #6
def test_read_empty_with_usecols(self): # See gh-12493 names = ['Dummy', 'X', 'Dummy_2'] usecols = names[1:2] # ['X'] # first, check to see that the response of # parser when faced with no provided columns # throws the correct error, with or without usecols errmsg = "No columns to parse from file" with tm.assert_raises_regex(EmptyDataError, errmsg): self.read_csv(StringIO('')) with tm.assert_raises_regex(EmptyDataError, errmsg): self.read_csv(StringIO(''), usecols=usecols) expected = DataFrame(columns=usecols, index=[0], dtype=np.float64) df = self.read_csv(StringIO(',,'), names=names, usecols=usecols) tm.assert_frame_equal(df, expected) expected = DataFrame(columns=usecols) df = self.read_csv(StringIO(''), names=names, usecols=usecols) tm.assert_frame_equal(df, expected)
Example #7
def detect_colspecs(self, n=100, skiprows=None): # Regex escape the delimiters delimiters = ''.join([r'\%s' % x for x in self.delimiter]) pattern = re.compile('([^%s]+)' % delimiters) rows = self.get_rows(n, skiprows) if not rows: raise EmptyDataError("No rows from which to infer column width") max_len = max(map(len, rows)) mask = np.zeros(max_len + 1, dtype=int) if self.comment is not None: rows = [row.partition(self.comment)[0] for row in rows] for row in rows: for m in pattern.finditer(row): mask[m.start():m.end()] = 1 shifted = np.roll(mask, 1) shifted[0] = 0 edges = np.where((mask ^ shifted) == 1)[0] edge_pairs = list(zip(edges[::2], edges[1::2])) return edge_pairs
Example #8
def parse_data(file_name, company_symbol): """ Returns data for the corresponding company :param file_name: name of the tar file :param company_symbol: company symbol :type file_name: str :type company_symbol: str :return: dataframe for the corresponding company data :rtype: pd.DataFrame """ tar = tarfile.open(file_name) try: price_report = pd.read_csv(tar.extractfile('prices.csv')) company_price_data = price_report[price_report['symbol'] == company_symbol] return company_price_data except (KeyError, pd_errors.EmptyDataError): return pd.DataFrame() # Getting the complete data for a given company
Example #9
def test_skiprows_callable(self): data = 'a\n1\n2\n3\n4\n5' skiprows = lambda x: x % 2 == 0 expected = DataFrame({'1': [3, 5]}) df = self.read_csv(StringIO(data), skiprows=skiprows) tm.assert_frame_equal(df, expected) expected = DataFrame({'foo': [3, 5]}) df = self.read_csv(StringIO(data), skiprows=skiprows, header=0, names=['foo']) tm.assert_frame_equal(df, expected) skiprows = lambda x: True msg = "No columns to parse from file" with tm.assert_raises_regex(EmptyDataError, msg): self.read_csv(StringIO(data), skiprows=skiprows) # This is a bad callable and should raise. msg = "by zero" skiprows = lambda x: 1 / 0 with tm.assert_raises_regex(ZeroDivisionError, msg): self.read_csv(StringIO(data), skiprows=skiprows)
Example #10
def test_read_empty_with_usecols(self): # See gh-12493 names = ['Dummy', 'X', 'Dummy_2'] usecols = names[1:2] # ['X'] # first, check to see that the response of # parser when faced with no provided columns # throws the correct error, with or without usecols errmsg = "No columns to parse from file" with tm.assert_raises_regex(EmptyDataError, errmsg): self.read_csv(StringIO('')) with tm.assert_raises_regex(EmptyDataError, errmsg): self.read_csv(StringIO(''), usecols=usecols) expected = DataFrame(columns=usecols, index=[0], dtype=np.float64) df = self.read_csv(StringIO(',,'), names=names, usecols=usecols) tm.assert_frame_equal(df, expected) expected = DataFrame(columns=usecols) df = self.read_csv(StringIO(''), names=names, usecols=usecols) tm.assert_frame_equal(df, expected)
Example #11
def test_read_empty_with_usecols(all_parsers, data, kwargs, expected): # see gh-12493 parser = all_parsers if expected is None: msg = "No columns to parse from file" with pytest.raises(EmptyDataError, match=msg): parser.read_csv(StringIO(data), **kwargs) else: result = parser.read_csv(StringIO(data), **kwargs) tm.assert_frame_equal(result, expected)
Example #12
def test_raise_on_no_columns(all_parsers, nrows): parser = all_parsers data = "\n" * nrows msg = "No columns to parse from file" with pytest.raises(EmptyDataError, match=msg): parser.read_csv(StringIO(data))
Example #13
def test_skip_rows_skip_all(all_parsers): parser = all_parsers data = "a\n1\n2\n3\n4\n5" msg = "No columns to parse from file" with pytest.raises(EmptyDataError, match=msg): parser.read_csv(StringIO(data), skiprows=lambda x: True)
Example #14
def test_zero_variables(datapath): # Check if the SAS file has zero variables (PR #18184) fname = datapath("io", "sas", "data", "zero_variables.sas7bdat") with pytest.raises(EmptyDataError): pd.read_sas(fname)
Example #15
def _parse(flavor, io, match, attrs, encoding, displayed_only, **kwargs): flavor = _validate_flavor(flavor) compiled_match = re.compile(match) # you can pass a compiled regex here # hack around python 3 deleting the exception variable retained = None for flav in flavor: parser = _parser_dispatch(flav) p = parser(io, compiled_match, attrs, encoding, displayed_only) try: tables = p.parse_tables() except Exception as caught: # if `io` is an io-like object, check if it's seekable # and try to rewind it before trying the next parser if hasattr(io, 'seekable') and io.seekable(): io.seek(0) elif hasattr(io, 'seekable') and not io.seekable(): # if we couldn't rewind it, let the user know raise ValueError('The flavor {} failed to parse your input. ' 'Since you passed a non-rewindable file ' 'object, we can\'t rewind it to try ' 'another parser. Try read_html() with a ' 'different flavor.'.format(flav)) retained = caught else: break else: raise_with_traceback(retained) ret = [] for table in tables: try: ret.append(_data_to_frame(data=table, **kwargs)) except EmptyDataError: # empty table continue return ret
Example #16
def test_raise_on_no_columns(self): # single newline data = "\n" pytest.raises(EmptyDataError, self.read_csv, StringIO(data)) # test with more than a single newline data = "\n\n\n" pytest.raises(EmptyDataError, self.read_csv, StringIO(data))
Example #17
def test_zero_variables(datapath): # Check if the SAS file has zero variables (PR #18184) fname = datapath("io", "sas", "data", "zero_variables.sas7bdat") with pytest.raises(EmptyDataError): pd.read_sas(fname)
Example #18
def _parse(flavor, io, match, attrs, encoding, displayed_only, **kwargs): flavor = _validate_flavor(flavor) compiled_match = re.compile(match) # you can pass a compiled regex here # hack around python 3 deleting the exception variable retained = None for flav in flavor: parser = _parser_dispatch(flav) p = parser(io, compiled_match, attrs, encoding, displayed_only) try: tables = p.parse_tables() except Exception as caught: # if `io` is an io-like object, check if it's seekable # and try to rewind it before trying the next parser if hasattr(io, 'seekable') and io.seekable(): io.seek(0) elif hasattr(io, 'seekable') and not io.seekable(): # if we couldn't rewind it, let the user know raise ValueError('The flavor {} failed to parse your input. ' 'Since you passed a non-rewindable file ' 'object, we can\'t rewind it to try ' 'another parser. Try read_html() with a ' 'different flavor.'.format(flav)) retained = caught else: break else: raise_with_traceback(retained) ret = [] for table in tables: try: ret.append(_data_to_frame(data=table, **kwargs)) except EmptyDataError: # empty table continue return ret
Example #19
def read(self, nrows=None): if (nrows is None) and (self.chunksize is not None): nrows = self.chunksize elif nrows is None: nrows = self.row_count if len(self.column_types) == 0: self.close() raise EmptyDataError("No columns to parse from file") if self._current_row_in_file_index >= self.row_count: return None m = self.row_count - self._current_row_in_file_index if nrows > m: nrows = m nd = (self.column_types == b'd').sum() ns = (self.column_types == b's').sum() self._string_chunk = np.empty((ns, nrows), dtype=np.object) self._byte_chunk = np.empty((nd, 8 * nrows), dtype=np.uint8) self._current_row_in_chunk_index = 0 p = Parser(self) p.read(nrows) rslt = self._chunk_to_dataframe() if self.index is not None: rslt = rslt.set_index(self.index) return rslt
Example #20
def get_csv_schema(filename, buildings): try: df = pd.read_csv(filename) except EmptyDataError: # csv file is empty return None schema = {"columns": {}} for column_name in df.columns: column_name = replace_repetitive_column_names(column_name, buildings) column_name = column_name.encode('ascii', 'ignore') schema["columns"][column_name] = get_column_schema(df[column_name]) return extract_df_schema(df, buildings)
Example #21
def test_read_empty_with_usecols(all_parsers, data, kwargs, expected): # see gh-12493 parser = all_parsers if expected is None: msg = "No columns to parse from file" with pytest.raises(EmptyDataError, match=msg): parser.read_csv(StringIO(data), **kwargs) else: result = parser.read_csv(StringIO(data), **kwargs) tm.assert_frame_equal(result, expected)
Example #22
def test_raise_on_no_columns(all_parsers, nrows): parser = all_parsers data = "\n" * nrows msg = "No columns to parse from file" with pytest.raises(EmptyDataError, match=msg): parser.read_csv(StringIO(data))
Example #23
def test_skip_rows_skip_all(all_parsers): parser = all_parsers data = "a\n1\n2\n3\n4\n5" msg = "No columns to parse from file" with pytest.raises(EmptyDataError, match=msg): parser.read_csv(StringIO(data), skiprows=lambda x: True)
Example #24
def test_zero_variables(datapath): # Check if the SAS file has zero variables (PR #18184) fname = datapath("io", "sas", "data", "zero_variables.sas7bdat") with pytest.raises(EmptyDataError): pd.read_sas(fname)
Example #25
def _parse(flavor, io, match, attrs, encoding, displayed_only, **kwargs): flavor = _validate_flavor(flavor) compiled_match = re.compile(match) # you can pass a compiled regex here # hack around python 3 deleting the exception variable retained = None for flav in flavor: parser = _parser_dispatch(flav) p = parser(io, compiled_match, attrs, encoding, displayed_only) try: tables = p.parse_tables() except Exception as caught: # if `io` is an io-like object, check if it's seekable # and try to rewind it before trying the next parser if hasattr(io, 'seekable') and io.seekable(): io.seek(0) elif hasattr(io, 'seekable') and not io.seekable(): # if we couldn't rewind it, let the user know raise ValueError('The flavor {} failed to parse your input. ' 'Since you passed a non-rewindable file ' 'object, we can\'t rewind it to try ' 'another parser. Try read_html() with a ' 'different flavor.'.format(flav)) retained = caught else: break else: raise_with_traceback(retained) ret = [] for table in tables: try: ret.append(_data_to_frame(data=table, **kwargs)) except EmptyDataError: # empty table continue return ret
Example #26
def get_hardware_utilization(self): """Retrieve GPU, CPU and memory utilization data. Get hardware utilization metrics for entire experiment as a single `pandas.DataFrame <//pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html>`_ object. Returned DataFrame has following columns (assuming single GPU with 0 index): * `x_ram` - time (in milliseconds) from the experiment start, * `y_ram` - memory usage in GB, * `x_cpu` - time (in milliseconds) from the experiment start, * `y_cpu` - CPU utilization percentage (0-100), * `x_gpu_util_0` - time (in milliseconds) from the experiment start, * `y_gpu_util_0` - GPU utilization percentage (0-100), * `x_gpu_mem_0` - time (in milliseconds) from the experiment start, * `y_gpu_mem_0` - GPU memory usage in GB. | If more GPUs are available they have their separate columns with appropriate indices (0, 1, 2, ...), for example: `x_gpu_util_1`, `y_gpu_util_1`. | The returned DataFrame may contain ``NaN`` s if one of the metrics has more values than others. Returns: :obj:`pandas.DataFrame` - DataFrame containing the hardware utilization metrics. Examples: The following values denote that after 3 seconds, the experiment used 16.7 GB of RAM * `x_ram` = 3000 * `y_ram` = 16.7 Assuming that `experiment` is an instance of :class:`~neptune.experiments.Experiment`: .. code:: python3 hardware_df = experiment.get_hardware_utilization() """ metrics_csv = self._backend.get_metrics_csv(self) try: return pd.read_csv(metrics_csv) except EmptyDataError: return pd.DataFrame()
Example #27
def _parse(flavor, io, match, attrs, encoding, **kwargs): flavor = _validate_flavor(flavor) compiled_match = re.compile(match) # you can pass a compiled regex here # hack around python 3 deleting the exception variable retained = None for flav in flavor: parser = _parser_dispatch(flav) p = parser(io, compiled_match, attrs, encoding) try: tables = p.parse_tables() except Exception as caught: retained = caught else: break else: raise_with_traceback(retained) ret = [] for table in tables: try: ret.append(_data_to_frame(data=table, **kwargs)) except EmptyDataError: # empty table continue return ret
Example #28
def test_raise_on_no_columns(self): # single newline data = "\n" pytest.raises(EmptyDataError, self.read_csv, StringIO(data)) # test with more than a single newline data = "\n\n\n" pytest.raises(EmptyDataError, self.read_csv, StringIO(data))
Example #29
def _parse(flavor, io, match, attrs, encoding, **kwargs): flavor = _validate_flavor(flavor) compiled_match = re.compile(match) # you can pass a compiled regex here # hack around python 3 deleting the exception variable retained = None for flav in flavor: parser = _parser_dispatch(flav) p = parser(io, compiled_match, attrs, encoding) try: tables = p.parse_tables() except Exception as caught: retained = caught else: break else: raise_with_traceback(retained) ret = [] for table in tables: try: ret.append(_data_to_frame(data=table, **kwargs)) except EmptyDataError: # empty table continue return ret
Example #30
def load_csv(self, idx=None, result=None): # load csv and fix sparse data. # csv can be in the middle of being written so we use try - except new_csv = None while new_csv is None: try: new_csv = pd.read_csv(self.full_csv_path) break except EmptyDataError: new_csv = None continue new_csv['Wall-Clock Time'] /= 60. new_csv = new_csv.interpolate() # remove signals which don't contain any values for k, v in new_csv.isna().all().items(): if v and k not in x_axis_options: del new_csv[k] # only fill the missing values that the previous interploation did not deal with (usually the ones before the # first update to the signal was made). We do so again by interpolation (averaging the previous and the next # values) new_csv = ((new_csv.fillna(method='bfill') + new_csv.fillna(method='ffill')) / 2).fillna(method='bfill').fillna( method='ffill') self.csv = new_csv self.last_modified = os.path.getmtime(self.full_csv_path) if idx is not None: result[idx] = (self.csv, self.last_modified)