Merge branch 'main' into tamarinvs19/python-codegen

Author: tamarinvs19

.github/workflows/collect-statistics.yml

@@ -62,6 +62,9 @@ env:
   data_path: monitoring/data
   monitoring_properties: monitoring/monitoring.properties
   push_script: monitoring/push_with_rebase.sh
+  PUSHGATEWAY_HOSTNAME: monitoring.utbot.org
+  PUSHGATEWAY_ADDITIONAL_PATH: /pushgateway-custom
+  PROM_ADDITIONAL_LABELS: /service/github
 
 jobs:
   setup_matrix:
@@ -136,7 +139,7 @@ jobs:
         id: insert
         shell: bash
         run: |
-          OUT_FILE="$data_path/$date-$branch-$short_commit-${{ matrix.value }}.json"
+          OUT_FILE="$data_path/$date-$short_commit-${{ matrix.value }}.json"
           echo "output=$OUT_FILE" >> $GITHUB_OUTPUT
 
           INPUTS=($(seq ${{ inputs.run_number }}))
@@ -184,7 +187,9 @@ jobs:
         if: ${{ inputs.send_to_grafana }}
         run: |
           python monitoring/prepare_metrics.py --stats_file $stats_file --output_file grafana_metrics.json
-          echo "TODO send metrics to grafana"
+          chmod +x scripts/project/json_to_prometheus.py
+          python3 scripts/project/json_to_prometheus.py grafana_metrics.json | curl -u "${{ secrets.PUSHGATEWAY_USER }}:${{ secrets.PUSHGATEWAY_PASSWORD }}" --data-binary @- "https://${PUSHGATEWAY_HOSTNAME}${PUSHGATEWAY_ADDITIONAL_PATH}/metrics/job/pushgateway-custom/instance/run-${{ matrix.value }}${PROM_ADDITIONAL_LABELS}"
+          echo "Please visit Grafana to check metrics: https://monitoring.utbot.org/d/m6bagaD4z/utbot-nightly-statistic"
         env:
           stats_file: ${{ steps.insert.outputs.output }}
 

.github/workflows/run-chosen-tests-from-branch.yml

@@ -30,6 +30,7 @@ on:
 env:
   # Environment variable setting gradle options.
   GRADLE_OPTS: "-XX:MaxHeapSize=2048m -Dorg.gradle.jvmargs='-XX:MaxHeapSize=2048m -XX:MaxPermSize=512m -Dorg.gradle.daemon=false' -Dorg.gradle.daemon=false"
+  PUSHGATEWAY_HOSTNAME: monitoring.utbot.org
 
 jobs:
     run-chosen-tests:
@@ -46,7 +47,7 @@ jobs:
         run: |
           echo Find your Prometheus metrics using label {instance=\"${GITHUB_RUN_ID}-${HOSTNAME}\"}
           chmod +x ./scripts/project/monitoring.sh
-          ./scripts/project/monitoring.sh ${{ secrets.PUSHGATEWAY_HOSTNAME }} ${{ secrets.PUSHGATEWAY_USER }} ${{ secrets.PUSHGATEWAY_PASSWORD }}
+          ./scripts/project/monitoring.sh ${PUSHGATEWAY_HOSTNAME} ${{ secrets.PUSHGATEWAY_USER }} ${{ secrets.PUSHGATEWAY_PASSWORD }}
           
       - name: Run chosen tests
         run: |

docs/OverallArchitecture.md

@@ -0,0 +1,179 @@
+# Overall Unit Test Bot Architecture
+
+Unit Test Bot overall architecture can be presented as following bird-eye view. Look below to check each component's purpose.
+
+```mermaid
+flowchart TB
+    subgraph Clients
+        direction LR
+        IntellijPlugin        
+        MavenPlugin["Maven/Gradle plugins"]
+        GithubAction-->MavenPlugin
+        ExternalJavaClient[\External Java Client\]
+        CLI
+        ContestEstimator    
+                     
+    end    
+
+    subgraph Facades
+        direction LR
+        EngineMain[[EngineMain]]
+        UtBotJavaApi[[UtBotJavaApi]]
+        GenerateTestsAndSarifReport[[GenerateTestsAndSarifReport]]       
+    end
+    IntellijPlugin--RD-->EngineMain
+    MavenPlugin-->GenerateTestsAndSarifReport
+    ExternalJavaClient-->UtBotJavaApi
+
+    subgraph API["Generation API"]
+        direction LR
+        TestCaseGenerator[[TestCaseGenerator]]
+        CodeGenerator[[CodeGenerator]]
+    end
+    Facades-->API
+    CLI-->API
+    ContestEstimator-->API
+    
+        
+
+    subgraph Components
+        direction LR
+        SymbolicEngine-->jlearch
+        SymbolicEngine-->ConcreteExecutor
+        Fuzzer-->ConcreteExecutor
+        SarifReport
+        Minimizer
+        CodeRenderer
+        Summaries
+        jlearch
+    end    
+    CodeGenerator-->CodeRenderer
+    TestCaseGenerator-->SymbolicEngine
+    TestCaseGenerator-->Fuzzer
+    TestCaseGenerator-->Minimizer
+    TestCaseGenerator-->Summaries
+    GenerateTestsAndSarifReport-->SarifReport
+
+    UTSettings((UTSettings))
+    UTSettings<--RD/local-->Components    
+    UTSettings<---->Clients
+    TestCaseGenerator--warmup-->ConcreteExecutor
+    ConcreteExecutor--RD-->InstrumentedProcess
+
+```
+
+## Typical interraction between components 
+
+Interaction diagram started from Intellij plugin UI is presented below
+```mermaid
+sequenceDiagram
+    autonumber
+    actor user as User
+    participant ij as Intellij plugin
+    participant engine as Engine process
+    participant concrete as Instrumented process
+    
+    user ->> ij: Invoke "Generate tests with UTBot"
+    ij ->> ij: Calculate methods, framework to show
+    ij ->> user: Show UI
+
+    break user clicked "Cancel"
+        user -->> user: Exit
+    end
+    user ->> ij: Click "Generate tests"
+    ij ->> ij: Calculate what jar need to be installed
+        
+    opt Some jars need to be installed  
+        ij ->> user: Do you accept new jars installed?
+        user -->> ij: [Accept/reject]
+        
+        alt Accepted
+            ij ->> ij: Install jars
+        end
+    end
+
+    ij ->> engine: Start process
+    activate engine
+    ij ->> engine: Setup ut context
+    
+    loop For all files
+        ij ->> engine: Generate UtExecutions
+        loop for all UtExecutions for method found by engine
+            engine ->> concrete: Run concretely            
+        end
+        engine --> engine: Minimize all tests for method
+        engine --> engine: Summaries for method
+    end
+    ij ->> engine: Render code
+    engine ->> ij: File with tests
+    deactivate engine
+
+    
+```
+
+# Clients
+
+### Intellij plugin
+> Module: [utbot-intellij](https://github.com/UnitTestBot/UTBotJava/tree/main/utbot-intellij)
+>
+> Purpose: UI interface for Java/Kotlin users
+
+
+TODO (Vassily Kudryashov)
+
+### Maven/gradle plugin
+
+TODO (Nikita Stroganov)
+
+### Github action
+
+TODO (Nikita Stroganov)
+
+### CLI
+
+TODO (Nikita Stroganov)
+
+### Contest estimator
+
+TODO (Rustam Sadykov)
+
+
+# Components
+
+### Symbolic engine
+TODO (Alexey Menshutin)
+
+### Concrete executor
+TODO (Sergey Pospelov)
+
+### Instrumented process
+TODO (Rustam Sadykov)
+
+### Code renderer
+TODO (Egor Kulikov)
+
+### Fuzzer
+TODO (Maxim Pelevin)
+
+### Minimizer
+TODO (Sergey Pospelov)
+
+### Summaries
+TODO (Alexey Zinoviev)
+
+### Sarif report
+TODO (Nikita Stroganov)
+
+
+
+# Cross-cutting subsystems
+
+### Logging
+TODO (Arteniy Kononov)
+
+### RD
+TODO (Arteniy Kononov)
+
+### UTSettings
+TODO (Vassily Kudryashov)
+

docs/StaticInitializersAnalysis.md

@@ -0,0 +1,73 @@
+# Symbolic analysis of static initializers
+
+## Problem
+
+Before the [Prohibit to set static fields from library classes](https://github.com/UnitTestBot/UTBotJava/pull/699) 
+change was implemented, every static field outside the `<clinit>` block (the so-called _meaningful_ static fields) 
+was stored in `modelBefore` and `modelAfter`. These _meaningful_ static fields were set (and reset for test isolation) during code generation. This led to explicit static field initializations, which looked unexpected for a user. For example, an `EMPTY` static field from the `Optional` class might be set for the following method under test
+
+```java
+class OptionalEmptyExample {
+    public java.util.Optional<Integer> optionalExample(boolean isEmpty) {
+        return isEmpty ? java.util.Optional.empty() : java.util.Optional.of(42);
+    }
+}
+```
+
+like:
+
+```java
+setStaticField(optionalClazz, "EMPTY", empty);
+```
+
+**Goal**: we should not set such kind of static fields with initializers.
+
+## Current solution
+
+Having merged [Prohibit to set static fields from library classes](https://github.com/UnitTestBot/UTBotJava/pull/699)
+, we now do not explicitly set the static fields of the classes from the so-called _trusted_ libraries (by default, 
+they are JDK packages). This behavior is guided by the `org.utbot.framework.
+UtSettings#getIgnoreStaticsFromTrustedLibraries` setting. Current solution possibly **leads to coverage regression** 
+and needs to be investigated: [Investigate coverage regression because of not setting static fields](https://github.com/UnitTestBot/UTBotJava/issues/716). 
+So, take a look at other ways to fix the problem.
+
+## Alternative solutions
+
+### Use concrete values as soft constraints _(not yet implemented)_
+
+The essence of the problem is assigning values to the static fields that should be set at runtime. To prevent it, 
+we can try to create models for the static fields according to their runtime values and filter out the static fields 
+that are equal to runtime values, using the following algorithm:
+
+1. Extract a concrete value for a static field.
+2. Create `UtModel` for this value and store it.
+3. Transform the produced model to soft constraints.
+4. Add them to the current symbolic state.
+5. Having resolved `stateBefore`, compare the resulting `UtModel` for the static field with the stored model and then drop the resulting model from `stateBefore` if they are equal.
+
+### Propagate information on the read static fields _(not yet implemented)_
+
+We can define the _meaningful_ static fields in a different way: we can mark the static fields as _meaningful_ if only they affect the method-under-test result. To decide if they do:
+
+- find out whether a given statement reads a specific static value or not and store this info,
+- while traversing the method graph, propagate this stored info to each of the following statements in a tree,
+- upon reaching the `return` statement of the method under test, mark all these read static fields as _meaningful_.
+
+### Filter out static methods: check if they affect `UtExecution` _(not yet implemented)_*
+Having collected all executions, we can analyze them and check whether the given static field affects the result of a current execution. Changing the static field value may have the same effect on every execution or no effect at all. It may also be required as an entry point during the executions (e.g., an _if_-statement as the first statement in the method under test):
+
+```java
+class AlwaysThrowingException {
+    public void throwIfMagic() {
+        if (ClassWithStaticField.staticField == 42) {
+            throw new RuntimeException("Magic number");
+        }
+    }
+}
+
+class ClassWithStaticField {
+    public final static int staticField = 42;
+} 
+```
+
+*This solution should only be used with the [propagation](#propagate-information-on-the-read-static-fields) solution.

gradle.properties

@@ -8,9 +8,8 @@ ideVersion=222.4167.29
 pythonIde=IC,IU,PC,PY
 jsIde=IU,PY,WS
 
-# In order to run Android Studion instead of Intellij Community,
-# specify the path to your Android Studio installation
-//androidStudioPath=your_path_to_android_studio
+# In order to run Android Studion instead of Intellij Community, specify the path to your Android Studio installation
+#androidStudioPath=your_path_to_android_studio
 
 #Version numbers: https://plugins.jetbrains.com/plugin/631-python/versions
 pythonCommunityPluginVersion=222.4167.37

monitoring/prepare_metrics.py

@@ -82,14 +82,14 @@ def build_metrics_from_data_array(metrics: List[dict], labels: dict) -> List[dic
     return result
 
 
-def build_metrics_from_target(target: dict, runner: str) -> List[dict]:
+def build_metrics_from_target(target: dict, run_id: str) -> List[dict]:
     result = []
     project = target["target"]
 
     result.extend(build_metrics_from_data_array(
         target["summarised"],
         {
-            "runner": runner,
+            "run_id": run_id,
             "project": project,
             "class": "All"
         }
@@ -100,7 +100,7 @@ def build_metrics_from_target(target: dict, runner: str) -> List[dict]:
         result.extend(build_metrics_from_data_array(
             class_item["data"],
             {
-                "runner": runner,
+                "run_id": run_id,
                 "project": project,
                 "class": class_name
             }
@@ -109,10 +109,10 @@ def build_metrics_from_target(target: dict, runner: str) -> List[dict]:
     return result
 
 
-def build_metrics_from_targets(targets: List[dict], runner: str) -> List[dict]:
+def build_metrics_from_targets(targets: List[dict], run_id: str) -> List[dict]:
     metrics = []
     for target in targets:
-        metrics.extend(build_metrics_from_target(target, runner))
+        metrics.extend(build_metrics_from_target(target, run_id))
     return metrics
 
 
@@ -131,11 +131,16 @@ def get_args():
     return args
 
 
+def extract_run_id(text: str):
+    idx = text.find('-')
+    return "run" + text[idx:]
+
+
 def main():
     args = get_args()
     stats = load(args.stats_file)
-    runner = stats["metadata"]["environment"]["host"]
-    metrics = build_metrics_from_targets(stats["targets"], runner)
+    run_id = extract_run_id(stats["metadata"]["source"]["id"])
+    metrics = build_metrics_from_targets(stats["targets"], run_id)
     metrics.sort(key=lambda x: x["metric"])
     with open(args.output_file, "w") as f:
         json.dump(metrics, f, indent=4)