import numpy as np
import pandas as pd
from ..aerosol2d import Aerosol2D
from .support.construct import build_2d
from .support.exceptions import FileFormatError
from .support.reading import read_cells, read_table, sniff
from .support.units import resolve_extra_columns
###############################################################################
[docs]
def load_ns_file(file: str, extra_data: bool = True) -> Aerosol2D:
"""Load a NanoScan SMPS (NS) CSV export and return it as an
:class:`Aerosol2D` number-size distribution with metadata.
Args:
file (str):
Path to the NanoScan CSV export file.
extra_data (bool, optional):
If ``True``, non-size-bin columns (e.g. status, density) are stored
in ``extra_data`` indexed by ``Datetime`` (with their units resolved
into ``column_units`` where recognisable). Defaults to ``True`` —
everything in the file is kept.
Returns:
Aerosol2D:
NanoScan size distributions with a datetime index, total
concentration, size-resolved bins, and associated metadata.
Raises:
FileNotFoundError:
If ``file`` does not exist or cannot be opened.
UnicodeDecodeError:
If the CSV file cannot be decoded using the encodings tried by
``sniff``.
ValueError:
If the reported data type/unit string cannot be mapped to a known
NanoScan format (e.g. dN, dS, dV, dM), or if the datetime column
cannot be parsed.
Notes:
Detailed description:
This loader is tailored to NanoScan SMPS CSV exports produced by
the TSI AIM instrument software. The file typically contains several
header rows, followed by a table with:
- a ``"Date Time"`` column,
- size-bin columns labelled by mid diameters (nm),
- additional metadata columns (e.g. particle density).
Internally, the function:
- Uses ``sniff`` to infer file encoding and field
delimiter.
- Reads the main data table, starting at the NanoScan data header
(``header=5``).
- Drops columns that are not part of the size distribution or time:
``"File Index"``, ``"Sample #"``, and ``"Total Conc"``.
- Extracts bin mid diameters from the first 13 size-bin column
headers and converts them to floats. From these, it computes bin
edges in nm using geometric means between adjacent mids, with
fixed outer edges at 10 nm and 420 nm.
- Renames ``"Date Time"`` to ``"Datetime"`` and parses timestamps
using the format ``"%Y/%m/%d %H:%M:%S"``.
- Selects the size-distribution columns (those matching the
bin-mid labels) as numeric data.
- Optionally collects all non-size-bin columns into ``extra_data``
when ``extra_data=True``, with ``Datetime`` as the index.
- Reads a header line to extract the instrument serial number.
- Reads the first data row to infer the data-type string
(e.g. ``"dN/dlogDp"``) and extracts particle density.
- Maps the data-type prefix (``"dN"``, ``"dS"``, ``"dV"``, ``"dM"``)
to a physical unit via a small lookup, raising a ``ValueError``
if the type is unknown.
- Computes ``"Total_conc"`` as the sum over all size bins for each
time step and concatenates ``Datetime``, ``Total_conc`` and the
size-bin columns into a single DataFrame.
- Creates an :class:`Aerosol2D` object from this DataFrame and
populates metadata:
- ``instrument`` set to ``"NS"``,
- ``bin_edges`` and ``bin_mids`` (rounded to one decimal place),
- ``density`` (float),
- ``serial_number``,
- ``unit`` (string) and ``dtype`` (raw type string from header).
- Calls ``_convert_to_number_concentration()`` followed by
:meth:`Aerosol2D.unnormalize_logdp` so that the final
distribution is expressed as number concentration per bin
(dN, cm⁻³) without ``/dlogDp`` normalisation.
- If ``extra_data=True``, attaches the non-size-bin columns as
``extra_data`` indexed by ``Datetime``.
Theory:
NanoScan exports can represent different moments of the particle
size distribution (number, surface, volume, mass) and may be
normalised by ``/dlogDp``. The raw type string typically encodes
this, for example:
- ``dN/dlogDp`` — number concentration per logarithmic diameter
interval,
- ``dS/dlogDp`` — surface-based moment,
- ``dV/dlogDp`` — volume-based moment,
- ``dM/dlogDp`` — mass-based moment.
From the two-letter prefix (``dN``, ``dS``, ``dV``, ``dM``), the
loader assigns the corresponding unit (cm⁻³, nm²/cm³, nm³/cm³,
µg/m³). The internal helper then converts any supported moment to
number concentration and removes the ``/dlogDp`` normalisation so
that the resulting distribution is directly comparable to other
number-size distributions in aerosoltools.
Examples:
Typical usage is to load a NanoScan CSV file and work directly
with the resulting number-size distribution:
.. code-block:: python
import aerosoltools as at
# Load NanoScan data (keep extra metadata columns)
ns = at.load_ns_file("data/NanoScan_export.csv", extra_data=True)
# Inspect the first few rows
print(ns.data.head())
# Check bin edges and metadata
print(ns.bin_edges)
print(ns.metadata)
# Plot a time-integrated or mean size distribution
fig, ax = ns.plot_psd()
"""
# Detect encoding and delimiter for the NanoScan export
encoding, delimiter = sniff(file)
# Load the full table from the main data header line
ns_df = read_table(
file, delimiter=delimiter, decimal=".", header=5, encoding=encoding
)
# Drop columns that are not part of the size distribution or time
ns_df.drop(columns=["File Index", "Sample #", "Total Conc"], inplace=True)
# Extract bin mid diameters from header and compute bin edges (nm)
bin_mids = np.array(ns_df.columns[1:14], dtype=float)
bin_edges = np.append(10.0, np.append(np.sqrt(bin_mids[1:] * bin_mids[:-1]), 420.0))
# Parse datetime column and rename to "Datetime"
ns_df.rename(columns={"Date Time": "Datetime"}, inplace=True)
ns_df["Datetime"] = pd.to_datetime(ns_df["Datetime"], format="%Y/%m/%d %H:%M:%S")
# Select size-distribution columns as numeric data
size_data = ns_df[bin_mids.astype(str)].copy()
# Keep all non-size-bin columns as extra_data (units resolved from headers)
column_units: dict[str, str] = {}
if extra_data:
ns_extra = ns_df.drop(columns=bin_mids.astype(str))
ns_extra.set_index("Datetime", inplace=True)
ns_extra, column_units = resolve_extra_columns(ns_extra)
else:
ns_extra = pd.DataFrame([])
# Read a header line for instrument serial number
with open(file, "r", encoding=encoding, newline="") as fh:
for _ in range(2):
fh.readline()
line = fh.readline()
fields = line.rstrip("\n").split(delimiter)
serial_number = fields[1].strip()
# Inspect first data row to infer dtype string and density
dtype_line = str(
read_cells(file, skip_header=5, encoding=encoding, delimiter=delimiter)
)
dtype = dtype_line.split(" ")[0]
density = ns_df["Particle Density (g/cc)"].iloc[0]
# Map data type prefix to physical unit
unit_dict = {
"dN": "cm⁻³",
"dS": "nm²/cm³",
"dV": "nm³/cm³",
"dM": "µg/m³",
}
try:
unit = unit_dict[dtype[:2]]
except KeyError as e:
raise FileFormatError(
"Unit and/or data type does not match the expected NanoScan format."
) from e
# Compute total concentration per time step from size bins
total_conc = size_data.sum(axis=1)
total_col = pd.DataFrame({"Total_conc": total_conc})
data_out = pd.concat([ns_df["Datetime"], total_col, size_data], axis=1)
# Instantiate, attach metadata, standardise to number and undo /dlogDp.
NS = build_2d(
data_out,
bin_edges=bin_edges.round(1),
bin_mids=bin_mids.round(1),
instrument="NS",
serial_number=serial_number,
unit=unit,
dtype=dtype,
density=float(density),
)
# Attach the kept extra channels and their resolved units
if extra_data:
NS._extra_data = ns_extra
NS._meta["column_units"] = column_units
return NS