/* * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as * published by the Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA * or visit www.oracle.com if you need additional information or have any * questions. * */ #include "cds/aotMetaspace.hpp" #include "cds/aotStreamedHeapLoader.hpp" #include "cds/aotThread.hpp" #include "cds/cdsConfig.hpp" #include "cds/filemap.hpp" #include "cds/heapShared.inline.hpp" #include "classfile/classLoaderDataShared.hpp" #include "classfile/javaClasses.inline.hpp" #include "classfile/stringTable.hpp" #include "classfile/vmClasses.hpp" #include "gc/shared/collectedHeap.inline.hpp" #include "gc/shared/oopStorage.inline.hpp" #include "gc/shared/oopStorageSet.inline.hpp" #include "logging/log.hpp" #include "memory/iterator.inline.hpp" #include "memory/oopFactory.hpp" #include "oops/access.inline.hpp" #include "oops/objArrayOop.inline.hpp" #include "oops/oop.inline.hpp" #include "runtime/globals.hpp" #include "runtime/globals_extension.hpp" #include "runtime/handles.inline.hpp" #include "runtime/java.hpp" #include "runtime/mutex.hpp" #include "runtime/thread.hpp" #include "utilities/bitMap.inline.hpp" #include "utilities/exceptions.hpp" #include "utilities/globalDefinitions.hpp" #include "utilities/stack.inline.hpp" #include "utilities/ticks.hpp" #include #if INCLUDE_CDS_JAVA_HEAP FileMapRegion* AOTStreamedHeapLoader::_heap_region; FileMapRegion* AOTStreamedHeapLoader::_bitmap_region; int* AOTStreamedHeapLoader::_roots_archive; OopHandle AOTStreamedHeapLoader::_roots; BitMapView AOTStreamedHeapLoader::_oopmap; bool AOTStreamedHeapLoader::_is_in_use; int AOTStreamedHeapLoader::_previous_batch_last_object_index; int AOTStreamedHeapLoader::_current_batch_last_object_index; int AOTStreamedHeapLoader::_current_root_index; size_t AOTStreamedHeapLoader::_allocated_words; bool AOTStreamedHeapLoader::_allow_gc; bool AOTStreamedHeapLoader::_objects_are_handles; size_t AOTStreamedHeapLoader::_num_archived_objects; int AOTStreamedHeapLoader::_num_roots; size_t AOTStreamedHeapLoader::_heap_region_used; bool AOTStreamedHeapLoader::_loading_all_objects; size_t* AOTStreamedHeapLoader::_object_index_to_buffer_offset_table; void** AOTStreamedHeapLoader::_object_index_to_heap_object_table; int* AOTStreamedHeapLoader::_root_highest_object_index_table; bool AOTStreamedHeapLoader::_waiting_for_iterator; bool AOTStreamedHeapLoader::_swapping_root_format; static uint64_t _early_materialization_time_ns = 0; static uint64_t _late_materialization_time_ns = 0; static uint64_t _final_materialization_time_ns = 0; static uint64_t _cleanup_materialization_time_ns = 0; static volatile uint64_t _accumulated_lazy_materialization_time_ns = 0; static Ticks _materialization_start_ticks; int AOTStreamedHeapLoader::object_index_for_root_index(int root_index) { return _roots_archive[root_index]; } int AOTStreamedHeapLoader::highest_object_index_for_root_index(int root_index) { return _root_highest_object_index_table[root_index]; } size_t AOTStreamedHeapLoader::buffer_offset_for_object_index(int object_index) { return _object_index_to_buffer_offset_table[object_index]; } oopDesc* AOTStreamedHeapLoader::archive_object_for_object_index(int object_index) { size_t buffer_offset = buffer_offset_for_object_index(object_index); address bottom = (address)_heap_region->mapped_base(); return (oopDesc*)(bottom + buffer_offset); } size_t AOTStreamedHeapLoader::buffer_offset_for_archive_object(oopDesc* archive_object) { address bottom = (address)_heap_region->mapped_base(); return size_t(archive_object) - size_t(bottom); } template BitMap::idx_t AOTStreamedHeapLoader::obj_bit_idx_for_buffer_offset(size_t buffer_offset) { if constexpr (use_coops) { return BitMap::idx_t(buffer_offset / sizeof(narrowOop)); } else { return BitMap::idx_t(buffer_offset / sizeof(HeapWord)); } } oop AOTStreamedHeapLoader::heap_object_for_object_index(int object_index) { assert(object_index >= 0 && object_index <= (int)_num_archived_objects, "Heap object reference out of index: %d", object_index); if (_objects_are_handles) { oop* handle = (oop*)_object_index_to_heap_object_table[object_index]; if (handle == nullptr) { return nullptr; } return NativeAccess<>::oop_load(handle); } else { return cast_to_oop(_object_index_to_heap_object_table[object_index]); } } void AOTStreamedHeapLoader::set_heap_object_for_object_index(int object_index, oop heap_object) { assert(heap_object_for_object_index(object_index) == nullptr, "Should only set once with this API"); if (_objects_are_handles) { oop* handle = Universe::vm_global()->allocate(); NativeAccess<>::oop_store(handle, heap_object); _object_index_to_heap_object_table[object_index] = (void*)handle; } else { _object_index_to_heap_object_table[object_index] = cast_from_oop(heap_object); } } int AOTStreamedHeapLoader::archived_string_value_object_index(oopDesc* archive_object) { assert(archive_object->klass() == vmClasses::String_klass(), "Must be an archived string"); address archive_string_value_addr = (address)archive_object + java_lang_String::value_offset(); return UseCompressedOops ? *(int*)archive_string_value_addr : (int)*(int64_t*)archive_string_value_addr; } static int archive_array_length(oopDesc* archive_array) { return *(int*)(address(archive_array) + arrayOopDesc::length_offset_in_bytes()); } static size_t archive_object_size(oopDesc* archive_object) { Klass* klass = archive_object->klass(); int lh = klass->layout_helper(); if (Klass::layout_helper_is_instance(lh)) { // Instance if (Klass::layout_helper_needs_slow_path(lh)) { return ((size_t*)(archive_object))[-1]; } else { return (size_t)Klass::layout_helper_size_in_bytes(lh) >> LogHeapWordSize; } } else if (Klass::layout_helper_is_array(lh)) { // Array size_t size_in_bytes; size_t array_length = (size_t)archive_array_length(archive_object); size_in_bytes = array_length << Klass::layout_helper_log2_element_size(lh); size_in_bytes += (size_t)Klass::layout_helper_header_size(lh); return align_up(size_in_bytes, (size_t)MinObjAlignmentInBytes) / HeapWordSize; } else { // Other return ((size_t*)(archive_object))[-1]; } } oop AOTStreamedHeapLoader::allocate_object(oopDesc* archive_object, markWord mark, size_t size, TRAPS) { assert(!archive_object->is_stackChunk(), "no such objects are archived"); NoJvmtiEventsMark njem; oop heap_object; Klass* klass = archive_object->klass(); if (klass->is_mirror_instance_klass()) { heap_object = Universe::heap()->class_allocate(klass, size, CHECK_NULL); } else if (klass->is_instance_klass()) { heap_object = Universe::heap()->obj_allocate(klass, size, CHECK_NULL); } else { assert(klass->is_array_klass(), "must be"); int length = archive_array_length(archive_object); bool do_zero = klass->is_objArray_klass(); heap_object = Universe::heap()->array_allocate(klass, size, length, do_zero, CHECK_NULL); } heap_object->set_mark(mark); return heap_object; } void AOTStreamedHeapLoader::install_root(int root_index, oop heap_object) { objArrayOop roots = objArrayOop(_roots.resolve()); OrderAccess::release(); // Once the store below publishes an object, it can be concurrently picked up by another thread without using the lock roots->obj_at_put(root_index, heap_object); } void AOTStreamedHeapLoader::TracingObjectLoader::wait_for_iterator() { if (JavaThread::current()->is_active_Java_thread()) { // When the main thread has bootstrapped past the point of allowing safepoints, // we can and indeed have to use safepoint checking waiting. AOTHeapLoading_lock->wait(); } else { // If we have no bootstrapped the main thread far enough, then we cannot and // indeed also don't need to perform safepoint checking waiting. AOTHeapLoading_lock->wait_without_safepoint_check(); } } // Link object after copying in-place template class AOTStreamedHeapLoader::InPlaceLinkingOopClosure : public BasicOopIterateClosure { private: oop _obj; LinkerT _linker; public: InPlaceLinkingOopClosure(oop obj, LinkerT linker) : _obj(obj), _linker(linker) { } virtual void do_oop(oop* p) { do_oop_work(p, (int)*(intptr_t*)p); } virtual void do_oop(narrowOop* p) { do_oop_work(p, *(int*)p); } template void do_oop_work(T* p, int object_index) { int p_offset = pointer_delta_as_int((address)p, cast_from_oop
(_obj)); oop pointee = _linker(p_offset, object_index); if (pointee != nullptr) { _obj->obj_field_put_access((int)p_offset, pointee); } } }; template void AOTStreamedHeapLoader::copy_payload_carefully(oopDesc* archive_object, oop heap_object, BitMap::idx_t header_bit, BitMap::idx_t start_bit, BitMap::idx_t end_bit, LinkerT linker) { using RawElementT = std::conditional_t; using OopElementT = std::conditional_t; BitMap::idx_t unfinished_bit = start_bit; BitMap::idx_t next_reference_bit = _oopmap.find_first_set_bit(unfinished_bit, end_bit); // Fill in heap object bytes while (unfinished_bit < end_bit) { assert(unfinished_bit >= start_bit && unfinished_bit < end_bit, "out of bounds copying"); // This is the address of the pointee inside the input stream size_t payload_offset = unfinished_bit - header_bit; RawElementT* archive_payload_addr = ((RawElementT*)archive_object) + payload_offset; RawElementT* heap_payload_addr = cast_from_oop(heap_object) + payload_offset; assert(heap_payload_addr >= cast_from_oop(heap_object) && (HeapWord*)heap_payload_addr < cast_from_oop(heap_object) + heap_object->size(), "Out of bounds copying"); if (next_reference_bit > unfinished_bit) { // Primitive bytes available size_t primitive_elements = next_reference_bit - unfinished_bit; size_t primitive_bytes = primitive_elements * sizeof(RawElementT); ::memcpy(heap_payload_addr, archive_payload_addr, primitive_bytes); unfinished_bit = next_reference_bit; } else { // Encountered reference RawElementT* archive_p = (RawElementT*)archive_payload_addr; OopElementT* heap_p = (OopElementT*)heap_payload_addr; int pointee_object_index = (int)*archive_p; int heap_p_offset = pointer_delta_as_int((address)heap_p, cast_from_oop
(heap_object)); // The object index is retrieved from the archive, not the heap object. This is // important after GC is enabled. Concurrent GC threads may scan references in the // heap for various reasons after this point. Therefore, it is not okay to first copy // the object index from a reference location in the archived object payload to a // corresponding location in the heap object payload, and then fix it up afterwards to // refer to a heap object. This is why this code iterates carefully over object references // in the archived object, linking them one by one, without clobbering the reference // locations in the heap objects with anything other than transitions from null to the // intended linked object. oop obj = linker(heap_p_offset, pointee_object_index); if (obj != nullptr) { heap_object->obj_field_put(heap_p_offset, obj); } unfinished_bit++; next_reference_bit = _oopmap.find_first_set_bit(unfinished_bit, end_bit); } } } template void AOTStreamedHeapLoader::copy_object_impl(oopDesc* archive_object, oop heap_object, size_t size, LinkerT linker) { if (!_allow_gc) { // Without concurrent GC running, we can copy incorrect object references // and metadata references into the heap object and then fix them up in-place. size_t payload_size = size - 1; HeapWord* archive_start = ((HeapWord*)archive_object) + 1; HeapWord* heap_start = cast_from_oop(heap_object) + 1; Copy::disjoint_words(archive_start, heap_start, payload_size); // In-place linking fixes up object indices from references of the heap object, // and patches them up to refer to objects. This can be done because we just copied // the payload of the object from the archive to the heap object, including the // reference object indices. However, this is only okay to do before the GC can run. // A concurrent GC thread might racingly read the object payload after GC is enabled. InPlaceLinkingOopClosure cl(heap_object, linker); heap_object->oop_iterate(&cl); HeapShared::remap_loaded_metadata(heap_object); return; } // When a concurrent GC may be running, we take care not to copy incorrect oops, // narrowOops or Metadata* into the heap objects. Transitions go from 0 to the // intended runtime linked values only. size_t word_scale = use_coops ? 2 : 1; using RawElementT = std::conditional_t; // Skip the markWord; it is set at allocation time size_t header_size = word_scale; size_t buffer_offset = buffer_offset_for_archive_object(archive_object); const BitMap::idx_t header_bit = obj_bit_idx_for_buffer_offset(buffer_offset); const BitMap::idx_t start_bit = header_bit + header_size; const BitMap::idx_t end_bit = header_bit + size * word_scale; BitMap::idx_t curr_bit = start_bit; // We are a bit paranoid about GC or other safepointing operations observing // shady metadata fields from the archive that do not point at real metadata. // We deal with this by explicitly reading the requested address from the // archive and fixing it to real Metadata before writing it into the heap object. HeapShared::do_metadata_offsets(heap_object, [&](int metadata_offset) { BitMap::idx_t metadata_field_idx = header_bit + (size_t)metadata_offset / sizeof(RawElementT); BitMap::idx_t skip = word_scale; assert(metadata_field_idx >= start_bit && metadata_field_idx + skip <= end_bit, "Metadata field out of bounds"); // Copy payload before metadata field copy_payload_carefully(archive_object, heap_object, header_bit, curr_bit, metadata_field_idx, linker); // Copy metadata field Metadata* const archive_metadata = *(Metadata**)(uintptr_t(archive_object) + (size_t)metadata_offset); Metadata* const runtime_metadata = archive_metadata != nullptr ? (Metadata*)(address(archive_metadata) + AOTMetaspace::relocation_delta()) : nullptr; assert(runtime_metadata == nullptr || AOTMetaspace::in_aot_cache(runtime_metadata), "Invalid metadata pointer"); DEBUG_ONLY(Metadata* const previous_metadata = heap_object->metadata_field(metadata_offset);) assert(previous_metadata == nullptr || previous_metadata == runtime_metadata, "Should not observe transient values"); heap_object->metadata_field_put(metadata_offset, runtime_metadata); curr_bit = metadata_field_idx + skip; }); // Copy trailing metadata after the last metadata word. This is usually doing // all the copying. copy_payload_carefully(archive_object, heap_object, header_bit, curr_bit, end_bit, linker); } void AOTStreamedHeapLoader::copy_object_eager_linking(oopDesc* archive_object, oop heap_object, size_t size) { auto linker = [&](int p_offset, int pointee_object_index) { oop obj = AOTStreamedHeapLoader::heap_object_for_object_index(pointee_object_index); assert(pointee_object_index == 0 || obj != nullptr, "Eager object loading should only encounter already allocated links"); return obj; }; if (UseCompressedOops) { copy_object_impl(archive_object, heap_object, size, linker); } else { copy_object_impl(archive_object, heap_object, size, linker); } } void AOTStreamedHeapLoader::TracingObjectLoader::copy_object_lazy_linking(int object_index, oopDesc* archive_object, oop heap_object, size_t size, Stack& dfs_stack) { auto linker = [&](int p_offset, int pointee_object_index) { dfs_stack.push({pointee_object_index, object_index, p_offset}); // The tracing linker is a bit lazy and mutates the reference fields in its traversal. // Returning null means don't link now. return oop(nullptr); }; if (UseCompressedOops) { copy_object_impl(archive_object, heap_object, size, linker); } else { copy_object_impl(archive_object, heap_object, size, linker); } } oop AOTStreamedHeapLoader::TracingObjectLoader::materialize_object_inner(int object_index, Stack& dfs_stack, TRAPS) { // Allocate object oopDesc* archive_object = archive_object_for_object_index(object_index); size_t size = archive_object_size(archive_object); markWord mark = archive_object->mark(); // The markWord is marked if the object is a String and it should be interned, // make sure to unmark it before allocating memory for the object. bool string_intern = mark.is_marked(); mark = mark.set_unmarked(); oop heap_object; if (string_intern) { int value_object_index = archived_string_value_object_index(archive_object); // Materialize the value object. (void)materialize_object(value_object_index, dfs_stack, CHECK_NULL); // Allocate and link the string. heap_object = allocate_object(archive_object, mark, size, CHECK_NULL); copy_object_eager_linking(archive_object, heap_object, size); assert(java_lang_String::value(heap_object) == heap_object_for_object_index(value_object_index), "Linker should have linked this correctly"); // Replace the string with interned string heap_object = StringTable::intern(heap_object, CHECK_NULL); } else { heap_object = allocate_object(archive_object, mark, size, CHECK_NULL); // Fill in object contents copy_object_lazy_linking(object_index, archive_object, heap_object, size, dfs_stack); } // Install forwarding set_heap_object_for_object_index(object_index, heap_object); return heap_object; } oop AOTStreamedHeapLoader::TracingObjectLoader::materialize_object(int object_index, Stack& dfs_stack, TRAPS) { if (object_index <= _previous_batch_last_object_index) { // The transitive closure of this object has been materialized; no need to do anything return heap_object_for_object_index(object_index); } if (object_index <= _current_batch_last_object_index) { // The AOTThread is currently materializing this object and its transitive closure; only need to wait for it to complete _waiting_for_iterator = true; while (object_index > _previous_batch_last_object_index) { wait_for_iterator(); } _waiting_for_iterator = false; // Notify the AOT thread if it is waiting for tracing to finish AOTHeapLoading_lock->notify_all(); return heap_object_for_object_index(object_index);; } oop heap_object = heap_object_for_object_index(object_index); if (heap_object != nullptr) { // Already materialized by mutator return heap_object; } return materialize_object_inner(object_index, dfs_stack, THREAD); } void AOTStreamedHeapLoader::TracingObjectLoader::drain_stack(Stack& dfs_stack, TRAPS) { while (!dfs_stack.is_empty()) { AOTHeapTraversalEntry entry = dfs_stack.pop(); int pointee_object_index = entry._pointee_object_index; oop pointee_heap_object = materialize_object(pointee_object_index, dfs_stack, CHECK); oop heap_object = heap_object_for_object_index(entry._base_object_index); if (_allow_gc) { heap_object->obj_field_put(entry._heap_field_offset_bytes, pointee_heap_object); } else { heap_object->obj_field_put_access(entry._heap_field_offset_bytes, pointee_heap_object); } } } oop AOTStreamedHeapLoader::TracingObjectLoader::materialize_object_transitive(int object_index, Stack& dfs_stack, TRAPS) { assert_locked_or_safepoint(AOTHeapLoading_lock); while (_waiting_for_iterator) { wait_for_iterator(); } auto handlized_materialize_object = [&](TRAPS) { oop obj = materialize_object(object_index, dfs_stack, CHECK_(Handle())); return Handle(THREAD, obj); }; Handle result = handlized_materialize_object(CHECK_NULL); drain_stack(dfs_stack, CHECK_NULL); return result(); } oop AOTStreamedHeapLoader::TracingObjectLoader::materialize_root(int root_index, Stack& dfs_stack, TRAPS) { int root_object_index = object_index_for_root_index(root_index); oop root = materialize_object_transitive(root_object_index, dfs_stack, CHECK_NULL); install_root(root_index, root); return root; } int oop_handle_cmp(const void* left, const void* right) { oop* left_handle = *(oop**)left; oop* right_handle = *(oop**)right; if (right_handle > left_handle) { return -1; } else if (left_handle > right_handle) { return 1; } return 0; } // The range is inclusive void AOTStreamedHeapLoader::IterativeObjectLoader::initialize_range(int first_object_index, int last_object_index, TRAPS) { for (int i = first_object_index; i <= last_object_index; ++i) { oopDesc* archive_object = archive_object_for_object_index(i); markWord mark = archive_object->mark(); bool string_intern = mark.is_marked(); if (string_intern) { int value_object_index = archived_string_value_object_index(archive_object); if (value_object_index == i + 1) { // Interned strings are eagerly materialized in the allocation phase, so there is // nothing else to do for interned strings here for the string nor its value array. i++; } continue; } size_t size = archive_object_size(archive_object); oop heap_object = heap_object_for_object_index(i); copy_object_eager_linking(archive_object, heap_object, size); } } // The range is inclusive size_t AOTStreamedHeapLoader::IterativeObjectLoader::materialize_range(int first_object_index, int last_object_index, TRAPS) { GrowableArrayCHeap lazy_object_indices(0); size_t materialized_words = 0; for (int i = first_object_index; i <= last_object_index; ++i) { oopDesc* archive_object = archive_object_for_object_index(i); markWord mark = archive_object->mark(); // The markWord is marked if the object is a String and it should be interned, // make sure to unmark it before allocating memory for the object. bool string_intern = mark.is_marked(); mark = mark.set_unmarked(); size_t size = archive_object_size(archive_object); materialized_words += size; oop heap_object = heap_object_for_object_index(i); if (heap_object != nullptr) { // Lazy loading has already initialized the object; we must not mutate it lazy_object_indices.append(i); continue; } if (!string_intern) { // The normal case; no lazy loading have loaded the object yet heap_object = allocate_object(archive_object, mark, size, CHECK_0); set_heap_object_for_object_index(i, heap_object); continue; } // Eagerly materialize interned strings to ensure that objects earlier than the string // in a batch get linked to the intended interned string, and not a copy. int value_object_index = archived_string_value_object_index(archive_object); bool is_normal_interned_string = value_object_index == i + 1; if (value_object_index < first_object_index) { // If materialized in a previous batch, the value should already be allocated and initialized. assert(heap_object_for_object_index(value_object_index) != nullptr, "should be materialized"); } else { // Materialize the value object. oopDesc* archive_value_object = archive_object_for_object_index(value_object_index); markWord value_mark = archive_value_object->mark(); size_t value_size = archive_object_size(archive_value_object); oop value_heap_object; if (is_normal_interned_string) { // The common case: the value is next to the string. This happens when only the interned // string points to its value character array. assert(value_object_index <= last_object_index, "Must be within this batch: %d <= %d", value_object_index, last_object_index); value_heap_object = allocate_object(archive_value_object, value_mark, value_size, CHECK_0); set_heap_object_for_object_index(value_object_index, value_heap_object); materialized_words += value_size; } else { // In the uncommon case, multiple strings point to the value of an interned string. // The string can then be earlier in the batch. assert(value_object_index < i, "surprising index"); value_heap_object = heap_object_for_object_index(value_object_index); } copy_object_eager_linking(archive_value_object, value_heap_object, value_size); } // Allocate and link the string. heap_object = allocate_object(archive_object, mark, size, CHECK_0); copy_object_eager_linking(archive_object, heap_object, size); assert(java_lang_String::value(heap_object) == heap_object_for_object_index(value_object_index), "Linker should have linked this correctly"); // Replace the string with interned string heap_object = StringTable::intern(heap_object, CHECK_0); set_heap_object_for_object_index(i, heap_object); if (is_normal_interned_string) { // Skip over the string value, already materialized i++; } } if (lazy_object_indices.is_empty()) { // Normal case; no sprinkled lazy objects in the root subgraph initialize_range(first_object_index, last_object_index, CHECK_0); } else { // The user lazy initialized some objects that are already initialized; we have to initialize around them // to make sure they are not mutated. int previous_object_index = first_object_index - 1; // Exclusive start of initialization slice for (int i = 0; i < lazy_object_indices.length(); ++i) { int lazy_object_index = lazy_object_indices.at(i); int slice_start_object_index = previous_object_index; int slice_end_object_index = lazy_object_index; if (slice_end_object_index - slice_start_object_index > 1) { // Both markers are exclusive initialize_range(slice_start_object_index + 1, slice_end_object_index - 1, CHECK_0); } previous_object_index = lazy_object_index; } // Process tail range if (last_object_index - previous_object_index > 0) { initialize_range(previous_object_index + 1, last_object_index, CHECK_0); } } return materialized_words; } bool AOTStreamedHeapLoader::IterativeObjectLoader::has_more() { return _current_root_index < _num_roots; } void AOTStreamedHeapLoader::IterativeObjectLoader::materialize_next_batch(TRAPS) { assert(has_more(), "only materialize if there is something to materialize"); int min_batch_objects = 128; int from_root_index = _current_root_index; int max_to_root_index = _num_roots - 1; int until_root_index = from_root_index; int highest_object_index; // Expand the batch size from one root, to N roots until we cross 128 objects in total for (;;) { highest_object_index = highest_object_index_for_root_index(until_root_index); if (highest_object_index - _previous_batch_last_object_index >= min_batch_objects) { break; } if (until_root_index == max_to_root_index) { break; } until_root_index++; } oop root = nullptr; // Materialize objects of necessary, representing the transitive closure of the root if (highest_object_index > _previous_batch_last_object_index) { while (_swapping_root_format) { // When the roots are being upgraded to use handles, it is not safe to racingly // iterate over the object; we must wait. Setting the current batch last object index // to something other than the previous batch last object index indicates to the // root swapping that there is current iteration ongoing. AOTHeapLoading_lock->wait(); } int first_object_index = _previous_batch_last_object_index + 1; _current_batch_last_object_index = highest_object_index; size_t allocated_words; { MutexUnlocker ml(AOTHeapLoading_lock, Mutex::_safepoint_check_flag); allocated_words = materialize_range(first_object_index, highest_object_index, CHECK); } _allocated_words += allocated_words; _previous_batch_last_object_index = _current_batch_last_object_index; if (_waiting_for_iterator) { // If tracer is waiting, let it know at the next point of unlocking that the root // set it waited for has been processed now. AOTHeapLoading_lock->notify_all(); } } // Install the root for (int i = from_root_index; i <= until_root_index; ++i) { int root_object_index = object_index_for_root_index(i); root = heap_object_for_object_index(root_object_index); install_root(i, root); ++_current_root_index; } } bool AOTStreamedHeapLoader::materialize_early(TRAPS) { Ticks start = Ticks::now(); // Only help with early materialization from the AOT thread if the heap archive can be allocated // without the need for a GC. Otherwise, do lazy loading until GC is enabled later in the bootstrapping. size_t bootstrap_max_memory = Universe::heap()->bootstrap_max_memory(); size_t bootstrap_min_memory = MAX2(_heap_region_used, 2 * M); size_t before_gc_materialize_budget_bytes = (bootstrap_max_memory > bootstrap_min_memory) ? bootstrap_max_memory - bootstrap_min_memory : 0; size_t before_gc_materialize_budget_words = before_gc_materialize_budget_bytes / HeapWordSize; log_info(aot, heap)("Max bootstrapping memory: %zuM, min bootstrapping memory: %zuM, selected budget: %zuM", bootstrap_max_memory / M, bootstrap_min_memory / M, before_gc_materialize_budget_bytes / M); while (IterativeObjectLoader::has_more()) { if (_allow_gc || _allocated_words > before_gc_materialize_budget_words) { log_info(aot, heap)("Early object materialization interrupted at root %d", _current_root_index); break; } IterativeObjectLoader::materialize_next_batch(CHECK_false); } _early_materialization_time_ns = (Ticks::now() - start).nanoseconds(); bool finished_before_gc_allowed = !_allow_gc && !IterativeObjectLoader::has_more(); return finished_before_gc_allowed; } void AOTStreamedHeapLoader::materialize_late(TRAPS) { Ticks start = Ticks::now(); // Continue materializing with GC allowed while (IterativeObjectLoader::has_more()) { IterativeObjectLoader::materialize_next_batch(CHECK); } _late_materialization_time_ns = (Ticks::now() - start).nanoseconds(); } void AOTStreamedHeapLoader::cleanup() { // First ensure there is no concurrent tracing going on while (_waiting_for_iterator) { AOTHeapLoading_lock->wait(); } Ticks start = Ticks::now(); // Remove OopStorage roots if (_objects_are_handles) { size_t num_handles = _num_archived_objects; // Skip the null entry oop** handles = ((oop**)_object_index_to_heap_object_table) + 1; // Sort the handles so that oop storage can release them faster qsort(handles, num_handles, sizeof(oop*), (int (*)(const void*, const void*))oop_handle_cmp); size_t num_null_handles = 0; for (size_t handles_remaining = num_handles; handles_remaining != 0; --handles_remaining) { oop* handle = handles[handles_remaining - 1]; if (handle == nullptr) { num_null_handles = handles_remaining; break; } NativeAccess<>::oop_store(handle, nullptr); } Universe::vm_global()->release(&handles[num_null_handles], num_handles - num_null_handles); } FREE_C_HEAP_ARRAY(void*, _object_index_to_heap_object_table); // Unmap regions FileMapInfo::current_info()->unmap_region(AOTMetaspace::hp); FileMapInfo::current_info()->unmap_region(AOTMetaspace::bm); _cleanup_materialization_time_ns = (Ticks::now() - start).nanoseconds(); log_statistics(); } void AOTStreamedHeapLoader::log_statistics() { uint64_t total_duration_us = (Ticks::now() - _materialization_start_ticks).microseconds(); const bool is_async = _loading_all_objects && !AOTEagerlyLoadObjects; const char* const async_or_sync = is_async ? "async" : "sync"; log_info(aot, heap)("start to finish materialization time: " UINT64_FORMAT "us", total_duration_us); log_info(aot, heap)("early object materialization time (%s): " UINT64_FORMAT "us", async_or_sync, _early_materialization_time_ns / 1000); log_info(aot, heap)("late object materialization time (%s): " UINT64_FORMAT "us", async_or_sync, _late_materialization_time_ns / 1000); log_info(aot, heap)("object materialization cleanup time (%s): " UINT64_FORMAT "us", async_or_sync, _cleanup_materialization_time_ns / 1000); log_info(aot, heap)("final object materialization time stall (sync): " UINT64_FORMAT "us", _final_materialization_time_ns / 1000); log_info(aot, heap)("bootstrapping lazy materialization time (sync): " UINT64_FORMAT "us", _accumulated_lazy_materialization_time_ns / 1000); uint64_t sync_time = _final_materialization_time_ns + _accumulated_lazy_materialization_time_ns; uint64_t async_time = _early_materialization_time_ns + _late_materialization_time_ns + _cleanup_materialization_time_ns; if (!is_async) { sync_time += async_time; async_time = 0; } log_info(aot, heap)("sync materialization time: " UINT64_FORMAT "us", sync_time / 1000); log_info(aot, heap)("async materialization time: " UINT64_FORMAT "us", async_time / 1000); uint64_t iterative_time = (uint64_t)(is_async ? async_time : sync_time); uint64_t materialized_bytes = _allocated_words * HeapWordSize; log_info(aot, heap)("%s materialized " UINT64_FORMAT "K (" UINT64_FORMAT "M/s)", async_or_sync, materialized_bytes / 1024, uint64_t(materialized_bytes * UCONST64(1'000'000'000) / M / iterative_time)); } void AOTStreamedHeapLoader::materialize_objects() { // We cannot handle any exception when materializing roots. Exits the VM. EXCEPTION_MARK // Objects are laid out in DFS order; DFS traverse the roots by linearly walking all objects HandleMark hm(THREAD); // Early materialization with a budget before GC is allowed MutexLocker ml(AOTHeapLoading_lock, Mutex::_safepoint_check_flag); materialize_early(CHECK); await_gc_enabled(); materialize_late(CHECK); // Notify materialization is done AOTHeapLoading_lock->notify_all(); cleanup(); } void AOTStreamedHeapLoader::switch_object_index_to_handle(int object_index) { oop heap_object = cast_to_oop(_object_index_to_heap_object_table[object_index]); if (heap_object == nullptr) { return; } oop* handle = Universe::vm_global()->allocate(); NativeAccess<>::oop_store(handle, heap_object); _object_index_to_heap_object_table[object_index] = handle; } void AOTStreamedHeapLoader::enable_gc() { if (AOTEagerlyLoadObjects && !IterativeObjectLoader::has_more()) { // Everything was loaded eagerly at early startup return; } // We cannot handle any exception when materializing roots. Exits the VM. EXCEPTION_MARK MutexLocker ml(AOTHeapLoading_lock, Mutex::_safepoint_check_flag); // First wait until no tracing is active while (_waiting_for_iterator) { AOTHeapLoading_lock->wait(); } // Lock further tracing from starting _waiting_for_iterator = true; // Record iterator progress int num_handles = (int)_num_archived_objects; // Lock further iteration from starting _swapping_root_format = true; // Then wait for the iterator to stop while (_previous_batch_last_object_index != _current_batch_last_object_index) { AOTHeapLoading_lock->wait(); } if (IterativeObjectLoader::has_more()) { // If there is more to be materialized, we have to upgrade the object index // to object mapping to use handles. If there isn't more to materialize, the // handle will no longer e used; they are only used to materialize objects. for (int i = 1; i <= num_handles; ++i) { // Upgrade the roots to use handles switch_object_index_to_handle(i); } // From now on, accessing the object table must be done through a handle. _objects_are_handles = true; } // Unlock tracing _waiting_for_iterator = false; // Unlock iteration _swapping_root_format = false; _allow_gc = true; AOTHeapLoading_lock->notify_all(); if (AOTEagerlyLoadObjects && IterativeObjectLoader::has_more()) { materialize_late(CHECK); cleanup(); } } void AOTStreamedHeapLoader::materialize_thread_object() { AOTThread::materialize_thread_object(); } void AOTStreamedHeapLoader::finish_materialize_objects() { Ticks start = Ticks::now(); if (_loading_all_objects) { MutexLocker ml(AOTHeapLoading_lock, Mutex::_safepoint_check_flag); // Wait for the AOT thread to finish while (IterativeObjectLoader::has_more()) { AOTHeapLoading_lock->wait(); } } else { assert(!AOTEagerlyLoadObjects, "sanity"); assert(_current_root_index == 0, "sanity"); // Without the full module graph we have done only lazy tracing materialization. // Ensure all roots are processed here by triggering root loading on every root. for (int i = 0; i < _num_roots; ++i) { get_root(i); } cleanup(); } _final_materialization_time_ns = (Ticks::now() - start).nanoseconds(); } void account_lazy_materialization_time_ns(uint64_t time, const char* description, int index) { AtomicAccess::add(&_accumulated_lazy_materialization_time_ns, time); log_debug(aot, heap)("Lazy materialization of %s: %d end (" UINT64_FORMAT " us of " UINT64_FORMAT " us)", description, index, time / 1000, _accumulated_lazy_materialization_time_ns / 1000); } // Initialize an empty array of AOT heap roots; materialize them lazily void AOTStreamedHeapLoader::initialize() { EXCEPTION_MARK _materialization_start_ticks = Ticks::now(); FileMapInfo::current_info()->map_bitmap_region(); _heap_region = FileMapInfo::current_info()->region_at(AOTMetaspace::hp); _bitmap_region = FileMapInfo::current_info()->region_at(AOTMetaspace::bm); assert(_heap_region->used() > 0, "empty heap archive?"); _is_in_use = true; // archived roots are at this offset in the stream. size_t roots_offset = FileMapInfo::current_info()->streamed_heap()->roots_offset(); size_t forwarding_offset = FileMapInfo::current_info()->streamed_heap()->forwarding_offset(); size_t root_highest_object_index_table_offset = FileMapInfo::current_info()->streamed_heap()->root_highest_object_index_table_offset(); _num_archived_objects = FileMapInfo::current_info()->streamed_heap()->num_archived_objects(); // The first int is the length of the array _roots_archive = ((int*)(((address)_heap_region->mapped_base()) + roots_offset)) + 1; _num_roots = _roots_archive[-1]; _heap_region_used = _heap_region->used(); // We can't retire a TLAB until the filler klass is set; set it to the archived object klass. CollectedHeap::set_filler_object_klass(vmClasses::Object_klass()); objArrayOop roots = oopFactory::new_objectArray(_num_roots, CHECK); _roots = OopHandle(Universe::vm_global(), roots); _object_index_to_buffer_offset_table = (size_t*)(((address)_heap_region->mapped_base()) + forwarding_offset); // We allocate the first entry for "null" _object_index_to_heap_object_table = NEW_C_HEAP_ARRAY(void*, _num_archived_objects + 1, mtClassShared); Copy::zero_to_bytes(_object_index_to_heap_object_table, (_num_archived_objects + 1) * sizeof(void*)); _root_highest_object_index_table = (int*)(((address)_heap_region->mapped_base()) + root_highest_object_index_table_offset); address start = (address)(_bitmap_region->mapped_base()) + _heap_region->oopmap_offset(); _oopmap = BitMapView((BitMap::bm_word_t*)start, _heap_region->oopmap_size_in_bits()); if (FLAG_IS_DEFAULT(AOTEagerlyLoadObjects)) { // Concurrency will not help much if there are no extra cores available. FLAG_SET_ERGO(AOTEagerlyLoadObjects, os::initial_active_processor_count() <= 1); } // If the full module graph is not available or the JVMTI class file load hook is on, we // will prune the object graph to not include cached objects in subgraphs that are not intended // to be loaded. _loading_all_objects = CDSConfig::is_using_full_module_graph() && !JvmtiExport::should_post_class_file_load_hook(); if (!_loading_all_objects) { // When not using FMG, fall back to tracing materialization FLAG_SET_ERGO(AOTEagerlyLoadObjects, false); return; } if (AOTEagerlyLoadObjects) { // Objects are laid out in DFS order; DFS traverse the roots by linearly walking all objects HandleMark hm(THREAD); // Early materialization with a budget before GC is allowed MutexLocker ml(AOTHeapLoading_lock, Mutex::_safepoint_check_flag); bool finished_before_gc_allowed = materialize_early(CHECK); if (finished_before_gc_allowed) { cleanup(); } } else { AOTThread::initialize(); } } oop AOTStreamedHeapLoader::materialize_root(int root_index) { Ticks start = Ticks::now(); // We cannot handle any exception when materializing a root. Exits the VM. EXCEPTION_MARK Stack dfs_stack; HandleMark hm(THREAD); oop result; { MutexLocker ml(AOTHeapLoading_lock, Mutex::_safepoint_check_flag); oop root = objArrayOop(_roots.resolve())->obj_at(root_index); if (root != nullptr) { // The root has already been materialized result = root; } else { // The root has not been materialized, start tracing materialization result = TracingObjectLoader::materialize_root(root_index, dfs_stack, CHECK_NULL); } } uint64_t duration = (Ticks::now() - start).nanoseconds(); account_lazy_materialization_time_ns(duration, "root", root_index); return result; } oop AOTStreamedHeapLoader::get_root(int index) { oop result = objArrayOop(_roots.resolve())->obj_at(index); if (result == nullptr) { // Materialize root result = materialize_root(index); } if (result == _roots.resolve()) { // A self-reference to the roots array acts as a sentinel object for null, // indicating that the root has been cleared. result = nullptr; } // Acquire the root transitive object payload OrderAccess::acquire(); return result; } void AOTStreamedHeapLoader::clear_root(int index) { // Self-reference to the roots array acts as a sentinel object for null, // indicating that the root has been cleared. objArrayOop(_roots.resolve())->obj_at_put(index, _roots.resolve()); } void AOTStreamedHeapLoader::await_gc_enabled() { while (!_allow_gc) { AOTHeapLoading_lock->wait(); } } void AOTStreamedHeapLoader::finish_initialization(FileMapInfo* static_mapinfo) { static_mapinfo->stream_heap_region(); } AOTMapLogger::OopDataIterator* AOTStreamedHeapLoader::oop_iterator(FileMapInfo* info, address buffer_start, address buffer_end) { class StreamedLoaderOopIterator : public AOTMapLogger::OopDataIterator { private: int _current; int _next; address _buffer_start; int _num_archived_objects; public: StreamedLoaderOopIterator(address buffer_start, int num_archived_objects) : _current(0), _next(1), _buffer_start(buffer_start), _num_archived_objects(num_archived_objects) { } AOTMapLogger::OopData capture(int dfs_index) { size_t buffered_offset = buffer_offset_for_object_index(dfs_index); address buffered_addr = _buffer_start + buffered_offset; oopDesc* raw_oop = (oopDesc*)buffered_addr; size_t size = archive_object_size(raw_oop); intptr_t target_location = (intptr_t)buffered_offset; uint32_t narrow_location = checked_cast(dfs_index); Klass* klass = raw_oop->klass(); address requested_addr = (address)buffered_offset; return { buffered_addr, requested_addr, target_location, narrow_location, raw_oop, klass, size, false }; } bool has_next() override { return _next <= _num_archived_objects; } AOTMapLogger::OopData next() override { _current = _next; AOTMapLogger::OopData result = capture(_current); _next = _current + 1; return result; } AOTMapLogger::OopData obj_at(narrowOop* addr) override { int dfs_index = (int)(*addr); if (dfs_index == 0) { return null_data(); } else { return capture(dfs_index); } } AOTMapLogger::OopData obj_at(oop* addr) override { int dfs_index = (int)cast_from_oop(*addr); if (dfs_index == 0) { return null_data(); } else { return capture(dfs_index); } } GrowableArrayCHeap* roots() override { GrowableArrayCHeap* result = new GrowableArrayCHeap(); for (int i = 0; i < _num_roots; ++i) { int object_index = object_index_for_root_index(i); result->append(capture(object_index)); } return result; } }; assert(_is_in_use, "printing before initializing?"); return new StreamedLoaderOopIterator(buffer_start, (int)info->streamed_heap()->num_archived_objects()); } #endif // INCLUDE_CDS_JAVA_HEAP