Creation d’une catalogue .zarr de l’archive GAMAR, avec virtualizarr / icechunk

from IPython.core.magic import register_cell_magic


@register_cell_magic
def skip(line, cell):
    return

import os
from functools import partial

import xarray as xr

import distributed

cluster = distributed.LocalCluster()
client = cluster.get_client()
client

import glob

from obstore.store import LocalStore
from virtualizarr import open_virtual_dataset
from virtualizarr.registry import ObjectStoreRegistry

# base_path = '/scale/project/lops-oh-fair2adapt/riomar/GAMAR/'
base_path = "/scale/project/lops-oh-fair2adapt/"

local_store = LocalStore()
registry = ObjectStoreRegistry({f"file://{base_path}": local_store})
# registry = ObjectStoreRegistry({})  # Pour test icechunk


# parser = KerchunkJSONParser()
# ou
from virtualizarr.parsers import HDFParser

parser = HDFParser()

base_path = "/scale/project/lops-oh-fair2adapt/riomar/GAMAR/"
# fpattern=base_path + 'GAMAR_1h_inst_Y2023M*.nc'
fpattern = base_path + "GAMAR_1h_inst_Y20*M*.nc"
flist = glob.glob(fpattern)[:]
len(flist)

dask_open_virtual_dataset = partial(
    open_virtual_dataset,
    loadable_variables=[
        "time_counter",
        "time_instant",
        "x_rho",
        "y_rho",
        "x_u",
        "x_v",
        "y_u",
        "y_v",
        "axis_nbounds",
    ],
    parser=parser,
    registry=registry,
    decode_times=True,
)
futures = client.map(dask_open_virtual_dataset, flist)
dss = client.gather(futures)
display(dss[:2])
display(dss[-2:])

%%skip version non dask
dss = [
    open_virtual_dataset(f,
        #concat_dim='time_counter',     # Dimension concaténation : time_counter
        #loadable_variables=['temp', 'nav_lat_rho', 'nav_lon_rho'],  # Seulement utiles
        loadable_variables=["time_counter","time_instant","x_rho","y_rho","x_u","x_v","y_u","y_v","axis_nbounds"],
        parser=parser,
        #combine='nested',           # compute nested = lent (hiérarchique), by_coords = rapide
        
        registry=registry,
                
        #coords="minimal",                # garde coords contenant time_counter seulement
        #data_vars="minimal",             # garde variables contenant time_counter seulement
        #compat="override",               # ignore différences entre fichiers (prend le 1er)
        #combine_attrs="drop_conflicts",  # supprime les attributs conflictuels
        decode_times=True,
    )
    for f in flist # ~5-10 secondes pour chaque fichier visiblement
]
dss

ds = xr.concat(
    dss,
    dim="time_counter",
    compat="override",
    coords="minimal",
    combine_attrs="drop_conflicts",
).set_coords(["time_counter_bounds", "time_instant_bounds"])
ds

outpath = "/scale/project/lops-oh-fair2adapt/fpaul/tmp/"
zarr_archive = os.path.join(outpath, "riomar.zarr")

import icechunk

storage = icechunk.local_filesystem_storage(zarr_archive)
storage

## code qui devrait marcher, exemple fourni sur :
# https://icechunk.io/en/stable/virtual/
# #creating-a-virtual-dataset-with-virtualizarr
config = icechunk.RepositoryConfig.default()
# config.set_virtual_chunk_container(
#     icechunk.VirtualChunkContainer(
#         "s3://mybucket/my/data/",
#         icechunk.s3_store(region="us-east-1"),
#     )
# )

## exemple de
# https://icechunk.io/en/stable/virtual/#example_1
# (cf. Local Filesystem.. ca utilise s3 dedans,
# mais probablement pas utilise en vrai)
# config.set_virtual_chunk_container(
#     icechunk.VirtualChunkContainer(
#         "s3://mybucket/my/data",
#         icechunk.local_filesystem_store(
#             "/path/to/my"
#         ),
#     )
# )
config.set_virtual_chunk_container(
    icechunk.VirtualChunkContainer(
        "file:///scale/project/lops-oh-fair2adapt/riomar/GAMAR/",
        icechunk.local_filesystem_store("/scale/project/lops-oh-fair2adapt/riomar"),
    )
)

credentials = credentials = icechunk.containers_credentials(
    {
        "file:///scale/project/lops-oh-fair2adapt/riomar/GAMAR/": None,
    }
)

assert not os.path.exists(zarr_archive), f"SHOULD NOT EXIST ! {zarr_archive}"

repo = icechunk.Repository.create(
    storage,
    config,
    authorize_virtual_chunk_access=credentials,
)
# cree le dossier dans storage, qui ne doit pas
# exister. Ici :
# /scale/project/lops-oh-fair2adapt/fpaul/tmp/
# riomar.zarr

session = repo.writable_session("main")
ds.virtualize.to_icechunk(
    session.store
)  # ca cree l'archive dans le storage indiquee plus haut
session.commit("My first virtual store!")

import os

os.environ["SHELL"] = "/bin/bash"
# %env SHELL=/bin/bash

!du -hs {zarr_archive}

!ls -alh {zarr_archive}

%%bash
cd /scale/project/lops-oh-fair2adapt/fpaul/tmp
./make_sed_web.sh
ls -alh *.json

import json
from itertools import islice

fichier = outfile

with open(fichier, "r", encoding="utf-8") as f:
    data = json.load(f)  # charge le JSON même sur une seule ligne
    texte = json.dumps(data, indent=2, ensure_ascii=False)

for ligne in islice(texte.splitlines(), 20):  # 40 premières lignes
    print(ligne)