[Feature Enhancement] Add tolerance record

graph_net/auto_fault_locator.py

@@ -12,7 +12,7 @@
 
 class AutoFaultLocator:
     def __init__(self, args):
-        self.log_file = os.path.abspath(args.log_file)
+        self.model_path_list = os.path.abspath(args.model_path_list)
         self.output_dir = os.path.abspath(args.output_dir)
         self.framework = args.framework
         self.decompose_method = args.decompose_method
@@ -23,20 +23,20 @@ def __init__(self, args):
         self.machine = args.machine
         self.port = args.port
 
-    def get_one_step_cmd(self, config_str):
-        config_b64 = convert_json_to_b64_string(config_str)
-        return [
+    def execute_one_step_cmd(self, test_config):
+        test_config_b64_str = convert_json_to_b64_string(test_config)
+        cmd = [
             sys.executable,
             "-m",
             "graph_net.subgraph_decompose_and_evaluation_step",
-            "--log-file",
-            self.log_file,
+            "--model-path-list",
+            self.model_path_list,
             "--output-dir",
             self.output_dir,
             "--framework",
             self.framework,
             "--test-config",
-            config_b64,
+            test_config_b64_str,
             "--decompose-method",
             self.decompose_method,
             "--tolerance",
@@ -45,49 +45,78 @@ def get_one_step_cmd(self, config_str):
             self.max_subgraph_size,
         ]
 
-    def run_remote_test_reference(self):
+        print(f"[AutoFaultLocator] Executing: {' '.join(cmd)}", flush=True)
+        result = subprocess.run(cmd, check=True, text=True)
+        return result
+
+    def run_test_reference_device(self, is_remote):
         print(
-            "\n>>> [Step 1] Run Remote Reference Device (Decomposition And Evaluation)\n"
+            "\n>>> [AutoFaultLocator 2/1] Run Test Reference Device (Decomposition And Evaluation)\n",
+            flush=True,
         )
 
-        test_remote_reference_device_config_str = {
-            "test_module_name": "test_remote_reference_device",
-            "test_remote_reference_device_arguments": {
+        test_module_name = (
+            "test_remote_reference_device" if is_remote else "test_reference_device"
+        )
+        test_reference_device_config = {
+            "test_module_name": test_module_name,
+            f"{test_module_name}_arguments": {
                 "model-path": None,
                 "reference-dir": None,
                 "compiler": "nope",
                 "device": self.reference_device,
-                "op-lib": "default",
                 "warmup": 5,
                 "trials": 20,
                 "seed": 123,
-                "machine": self.machine,
-                "port": self.port,
             },
         }
-
-        cmd = self.get_one_step_cmd(test_remote_reference_device_config_str)
-        print(f"Executing: {' '.join(cmd)}")
-        result = subprocess.run(cmd, check=True, text=True)
+        if args.framework == "torch":
+            test_reference_device_config[f"{test_module_name}_arguments"].update(
+                {"op-lib": "default"}
+            )
+        if is_remote:
+            test_reference_device_config[f"{test_module_name}_arguments"].update(
+                {
+                    "machine": self.machine,
+                    "port": self.port,
+                }
+            )
+
+        result = self.execute_one_step_cmd(test_reference_device_config)
         assert (
             result.returncode == 0
         ), f"Run Remote Reference Device failed with return code {result.returncode}"
 
-    def run_local_test_target(self):
-        print("\n>>> [Step 2] Run Local Target Device (Evaluation And Analysis)\n")
+    def run_test_target_device(self, is_remote):
+        print(
+            "\n>>> [AutoFaultLocator 2/2] Run Test Target Device (Evaluation And Analysis)\n",
+            flush=True,
+        )
 
-        test_target_device_config_str = {
-            "test_module_name": "test_target_device",
-            "test_target_device_arguments": {
+        test_module_name = (
+            "test_remote_target_device" if is_remote else "test_target_device"
+        )
+        test_target_device_config = {
+            "test_module_name": test_module_name,
+            f"{test_module_name}_arguments": {
                 "model-path": None,
                 "reference-dir": None,
+                "compiler": "nope",
                 "device": self.target_device,
+                "warmup": 5,
+                "trials": 20,
+                "seed": 123,
             },
         }
-
-        cmd = self.get_one_step_cmd(test_target_device_config_str)
-        print(f"Executing: {' '.join(cmd)}")
-        result = subprocess.run(cmd, check=True, text=True)
+        if is_remote:
+            test_target_device_config[f"{test_module_name}_arguments"].update(
+                {
+                    "machine": self.machine,
+                    "port": self.port,
+                }
+            )
+
+        result = self.execute_one_step_cmd(test_target_device_config)
         assert (
             result.returncode == 0
         ), f"Run Local Target Device failed with return code {result.returncode}"
@@ -114,16 +143,16 @@ def analyze_and_decide_next(self):
 def main(args):
     locator = AutoFaultLocator(args)
     while True:
-        locator.run_remote_test_reference()
-        locator.run_local_test_target()
+        locator.run_test_reference_device(is_remote=False)
+        locator.run_test_target_device(is_remote=True)
         should_continue = locator.analyze_and_decide_next()
         if not should_continue:
             break
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
-    parser.add_argument("--log-file", type=str, required=True)
+    parser.add_argument("--model-path-list", type=str, required=True)
     parser.add_argument("--output-dir", type=str, required=True)
     parser.add_argument(
         "--framework", type=str, choices=["paddle", "torch"], required=True

graph_net/config/paddle_samples_list.txt

@@ -272,7 +272,6 @@ paddle_samples/PaddleX/ResNet101_vd
 paddle_samples/PaddleNLP/roformer_v2_chinese_char_small
 paddle_samples/PaddleX/Mask-RT-DETR-X/subgraph_0
 paddle_samples/PaddleX/Mask-RT-DETR-X/subgraph_1
-paddle_samples/PaddleX/PP-ShiTuV2_det/subgraph_1
 paddle_samples/PaddleX/MobileNetV4_conv_medium/subgraph_0
 paddle_samples/PaddleX/PicoDet-S/subgraph_9
 paddle_samples/PaddleX/PP-HGNetV2-B5/subgraph_0

graph_net/paddle/prologue_subgraph_unittest_generator.py

@@ -270,22 +270,42 @@ def _get_output_shapes(outs):
         )
         self.assertTrue(shape_match, f"Shape of outputs are not matched ({reference_shapes=} vs {target_shapes=}).")
 
-    def check_results(self, reference_outputs, target_outputs):
-        def _convert_to_numpy(out):
-            if out.dtype not in [paddle.float32, paddle.float64]:
-                return out.cast("float32").numpy()
-            else:
-                return out.numpy()
+    def convert_to_numpy(self, out):
+        if out.dtype not in [paddle.float32, paddle.float64]:
+            return out.cast("float32").numpy()
+        else:
+            return out.numpy()
+
+    def get_all_cmp_results(self, reference_outputs, target_outputs, name):
+        def _get_cmp_allclose(cmp_arrays, tolerance):
+            return [
+                int(np.allclose(actual, desired, atol=atol, rtol=rtol))
+                for dtype, actual, desired in cmp_arrays
+                for (atol, rtol) in [tolerance_generator(tolerance, dtype)]
+            ]
 
+        cmp_arrays = [
+            (reference.dtype, self.convert_to_numpy(reference), self.convert_to_numpy(target)) 
+            for reference, target in zip(reference_outputs, target_outputs)]
+        for tolerance in range(-10, 2):
+            cmp_results = _get_cmp_allclose(cmp_arrays, tolerance)
+            is_correct = all(x == 1 for x in cmp_results)
+            cmp_results_str = " ".join(str(v) for v in cmp_results)
+            print(f"{name}, tolerance: {tolerance:3d}, allclose: {is_correct}, cmp_result: {cmp_results_str}")
+        print()
+
+    def check_results(self, reference_outputs, target_outputs, name):
         assert len(reference_outputs) == len(target_outputs), f"The number of outputs is not equal ({len(reference_outputs)=} vs {len(target_outputs)})."
         self.check_dtypes(reference_outputs, target_outputs)
         self.check_shapes(reference_outputs, target_outputs)
 
+        self.get_all_cmp_results(reference_outputs, target_outputs, name)
+
         for reference, target in zip(reference_outputs, target_outputs):
             atol, rtol = tolerance_generator(self.tolerance, reference.dtype)
             np.testing.assert_allclose(
-                actual=_convert_to_numpy(target),
-                desired=_convert_to_numpy(reference),
+                actual=self.convert_to_numpy(target),
+                desired=self.convert_to_numpy(reference),
                 atol=atol,
                 rtol=rtol,
             )
@@ -302,7 +322,7 @@ def test_separated(self):
             print(f"Load prologue output tensors from {prologue_output_path}")
             prologue_reference_outputs = paddle.load(prologue_output_path)
             with self.subTest(name="check_prologue_outputs"):
-                self.check_results(prologue_reference_outputs, prologue_outputs)
+                self.check_results(prologue_reference_outputs, prologue_outputs, name="check_prologue_outputs")
 
         test_output_path = os.path.join(self.reference_dir, "{{graph_module_desc.model_name}}_separated_reference.pdout")
         test_outputs = self.run_suspect_layer(prologue_reference_outputs)
@@ -313,7 +333,7 @@ def test_separated(self):
             print(f"Load test output tensors on reference device from {test_output_path}.")
             test_reference_outputs = paddle.load(test_output_path)
             with self.subTest(name="check_suspect_outputs"):
-                self.check_results(test_reference_outputs, test_outputs)
+                self.check_results(test_reference_outputs, test_outputs, name="check_suspect_outputs")
 
     def test_combined(self):
         paddle.seed(self.runtime_seed)
@@ -326,7 +346,7 @@ def test_combined(self):
             print(f"Load test output tensors on reference device from {test_output_path}.")
             test_reference_outputs = paddle.load(test_output_path)
             with self.subTest(name="check_combined_outputs"):
-                self.check_results(test_reference_outputs, test_outputs)
+                self.check_results(test_reference_outputs, test_outputs, name="check_combined_outputs")
 
 
 if __name__ == "__main__":

graph_net/paddle/test_compiler.py

@@ -78,8 +78,7 @@ def load_class_from_file(file_path: str, class_name: str):
 
     with open(file_path, "r", encoding="utf-8") as f:
         original_code = f.read()
-    import_stmt = "import paddle"
-    modified_code = f"{import_stmt}\n{original_code}"
+    modified_code = utils.rewrite_model(original_code)
     spec = importlib.util.spec_from_loader(module_name, loader=None)
     module = importlib.util.module_from_spec(spec)
     sys.modules[module_name] = module