BOP-ASK: Object Interaction Reasoning for Vision Language Models (CVPR 2026).

bopask-github/
├── config/
│   └── dataset_config.json                # Per-dataset depth scales, resolutions, methods
│
├── generate_surface_masks.py              # Step 1:  Molmo + SAM2 surface segmentation
├── render_objects_and_descriptions.py     # Step 2:  PyVista renders + GPT descriptions
├── export_bounding_boxes.py               # Step 3:  3D bbox export per scene
├── run_all_extrinsic_estimations.py       # Step 4:  Camera extrinsic estimation (wrapper)
├── bop2m2t2.py                            # Step 5:  BOP -> M2T2 format conversion
├── generate_all_trajectories_parallel.py  # Step 6:  RRT pick-and-place trajectories (parallel)
├── run_all_grasp_generation.py            # Step 7:  M2T2 6-DoF grasp generation
├── filter_grasp_collisions.py             # Step 8:  3D grasp collision filtering
├── generate_question_templates.py         # Step 9:  Pre-generate question variations
├── generate_batch_qa_dataset.py           # Step 10: Final JSONL + image generation (emits debug visualizations)
│
├── utils/                                 # Shared helpers imported by the pipeline
│   ├── bop_utils.py                       # OpenAI client, text drawing
│   ├── estimate_cam2world.py              # Core extrinsic computation (invoked by step 4)
│   ├── generate_all_trajectories.py       # Sequential trajectory generation + helpers (imported by step 6)
│   ├── plan_motion_rrt.py                 # Core RRT planner (imported by step 6)
│   ├── grasp_vis_utils.py                 # 2D grasp point extraction (used by steps 8 and 10)
│   └── visualize_3d_boxes.py              # BOP bbox visualizer class (used by steps 3 and 10)
│
├── M2T2/                                  # Vendored: M2T2 grasp prediction model
├── question_templates/                    # Cached question templates (per dataset/split, git-ignored)
├── requirements.txt                       # pip dependencies (see Environment Setup)
└── download_bop_datasets.sh               # Convenience script to fetch BOP datasets

bop/
└── hope/
    ├── camera.json                   # Global camera intrinsics
    ├── models/
    │   ├── obj_XXXXXX.ply            # 3D meshes
    │   ├── obj_XXXXXX.png            # Textures (optional)
    │   └── models_info.json          # Per-model bounding box metadata
    ├── models_eval/                  # Simplified models for evaluation
    └── val/                          # Split directory
        └── 000001/                   # Scene directory
            ├── rgb/                  # RGB images: 000000.png, 000001.png, ...
            ├── depth/                # 16-bit PNG depth maps (values in mm)
            ├── mask/                 # Per-object binary masks: {frame}_{idx}.png
            ├── mask_visib/           # Visible-part masks
            ├── scene_camera.json     # Per-frame camera intrinsics
            ├── scene_gt.json         # Ground-truth 6D poses
            └── scene_gt_info.json    # Additional GT metadata

bop/hope/val/surface_masks/
    ├── 1_0.png       # scene 1, frame 0 — binary mask (0/255), same resolution as RGB
    ├── 1_1.png
    └── ...

renders_pyvista_hope/
    ├── renders/
    │   └── obj_000001.png             # 2x2 grid of 4 views per object
    └── object_descriptions.json       # {obj_key: {render_path, description, model_info}}

bop/hope/object_descriptions.json      # Copy for downstream scripts

bop/hope/val/000001/
    ├── bboxes_000001.json          # Per-frame bounding box data
    ├── scene_gt.json               # Modified with unique object IDs
    └── scene_gt_original.json      # Backup of original scene_gt

bop/hope/val/000001/
    ├── T_wc_0.txt                   # 4x4 float matrix (or "FAILED" on first line)
    ├── T_wc_1.txt
    └── extrinsic_visualizations/    # Debug overlays
        ├── object_up_vectors_0.png
        └── fitted_ransac_plane_0.png

bop/hope/val/converted-bop/
    └── scene_000001_frame_000000/
        ├── rgb.png                   # Resized to 1280x720
        ├── depth.npy                 # float32, metres, shape (720, 1280)
        ├── seg.png                   # uint16 semantic mask (0=bg, obj_label=fg)
        └── meta_data.pkl            # dict: camera_pose (4x4), intrinsics (3x3),
                                     #       label_map ({"obj_2a": 297, ...}),
                                     #       scene_bounds (None), ee_pose (None)

bop/hope/trajectories_val.json

bop/hope/val/converted-bop/scene_000001_frame_000000/
    └── scene_state_complete.json     # Per-object: grasp poses, point cloud, confidence

question_templates/question_templates_hope_val.json

hope-val-hf/
    ├── val.jsonl                     # ShareGPT-format conversations
    ├── images/                       # One RGB image per scene-frame
    │   └── scene_000001_frame_000000.png
    └── debug-images/                 # QA overlays for verification

bop/
└── hope/
    ├── camera.json
    ├── object_descriptions.json           # From step 2
    ├── trajectories_val.json              # From step 6
    ├── models/
    ├── models_eval/
    └── val/
        ├── 000001/                        # Scene directory
        │   ├── rgb/
        │   ├── depth/
        │   ├── mask/
        │   ├── mask_visib/
        │   ├── scene_camera.json
        │   ├── scene_gt.json              # Modified with unique IDs (step 3)
        │   ├── scene_gt_original.json     # Backup (step 3)
        │   ├── bboxes_000001.json         # 3D bbox data (step 3)
        │   ├── T_wc_0.txt                 # Extrinsics (step 4)
        │   ├── T_wc_1.txt
        │   └── extrinsic_visualizations/  # Debug images (step 4)
        ├── surface_masks/                 # Surface masks (step 1)
        │   ├── 1_0.png
        │   └── ...
        └── converted-bop/                 # M2T2 format (step 5)
            └── scene_000001_frame_000000/
                ├── rgb.png
                ├── depth.npy
                ├── seg.png
                ├── meta_data.pkl
                └── scene_state_complete.json  # Grasps (steps 7-8)

hope-val-hf/                               # Final output (step 10)
    ├── val.jsonl
    ├── images/
    └── debug-images/

renders_pyvista_hope/                      # Object renders (step 2)
    ├── renders/
    └── object_descriptions.json

question_templates/                        # Cached templates (step 9)
    └── question_templates_hope_val.json