Merge pull request #19789 from mpirvu/disclaim_fix

Fixes to code cache disclaiming

Author: dsouzai

runtime/compiler/runtime/J9CodeCache.cpp

@@ -153,38 +153,76 @@ J9::CodeCache::initialize(TR::CodeCacheManager *manager,
       {
       J9JavaVM * javaVM = jitConfig->javaVM;
       PORT_ACCESS_FROM_JAVAVM(javaVM); // for j9vmem_supported_page_sizes
+      uintptr_t round = (uintptr_t)(j9vmem_supported_page_sizes()[0] - 1);
+
+      // The code cache segment start must be page aligned because it was allocated with mmap
+      // _warmCodeAllocBase may not be at the very beginning of the segment though (and not page alinged)
+      // Align on page boundary going backwards if needed.
+      uintptr_t warmSectionStart = (uintptr_t)_warmCodeAllocBase & ~round;
+      // The cold section may be followed by trampolines and helpers, so it may not be page aligned.
+      // Align it going forward. This will not go past the end of the segment which is page aligned.
+      uintptr_t coldSectionEnd = ((uintptr_t)_coldCodeAllocBase  + round) & ~round;
+      // Find the split point between the warm area (using large pages) and the cold area (using small pages).
+      // Experiments have shown that warm/cold code is about 50/50, so we need to find the middle point.
+      uintptr_t middle  = warmSectionStart + (coldSectionEnd - warmSectionStart) / 2;
+      uintptr_t coldSectionStart = middle;
+      // Since we want to use large pages for the warm area, its end needs to be large page aligned.
+      // We cannot determine the size of the THP page with j9vmem_supported_page_sizes(),
+      // but we know that on x86, the size of a THP page is 2 MB. Round up/down as needed.
+#if defined(TR_TARGET_X86)
+      static const uintptr_t THP_SIZE = 2 * 1024 * 1024; // 2 MB
+#elif defined(TR_TARGET_S390)
+      static const uintptr_t THP_SIZE = 1 * 1024 * 1024; // 1 MB
+#else
+      // Power has 64 KB and 16 MB pages (16 MB is too large to be useful for disclaiming)
+      // ARM can have many sizes for its large pages: 64 K, 1 MB, 2 MB, 16 MB)
+      static const uintptr_t THP_SIZE = 65536; // 64K
+#endif
+      static const uintptr_t ROUNDING_VALUE = THP_SIZE/2 - 1;
+      if (codeCacheSegment->segmentTop() - codeCacheSegment->segmentBase() >= 2 * THP_SIZE)
+         {
+         coldSectionStart = (middle + ROUNDING_VALUE) & ~ROUNDING_VALUE;
+         }
+      TR_ASSERT_FATAL(coldSectionEnd > coldSectionStart, "A code cache can't be smaller than a page");
+      _smallPageAreaStart = (uint8_t *)coldSectionStart;
+      _smallPageAreaEnd = (uint8_t *)coldSectionEnd;
 
-      uint8_t *middle  = _warmCodeAlloc + (_coldCodeAllocBase - _warmCodeAlloc) / 2;
-      size_t round = j9vmem_supported_page_sizes()[0] - 1;
-      middle = (uint8_t *)(((size_t)(middle + round)) & ~round);
-
-      TR_ASSERT_FATAL(_coldCodeAlloc > middle, "A code cache can't be smaller than a page");
+      size_t coldCacheSize = coldSectionEnd - coldSectionStart;
 
-      size_t coldCacheSize = _coldCodeAlloc - middle;
-      coldCacheSize = (coldCacheSize + round) & ~round;
+      if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerbosePerformance))
+         TR_VerboseLog::writeLineLocked(TR_Vlog_INFO, "Code cache warm area %p - %p (size=%zu); cold area %p - %p (size=%zu)",
+                                        (uint8_t*)warmSectionStart, _smallPageAreaStart, _smallPageAreaStart - (uint8_t*)warmSectionStart,
+                                        _smallPageAreaStart, _smallPageAreaEnd, coldCacheSize);
 
-      if (madvise(middle, coldCacheSize, MADV_NOHUGEPAGE) != 0)
+      if (madvise(_smallPageAreaStart, coldCacheSize, MADV_NOHUGEPAGE) != 0)
          {
          const char *error = strerror(errno);
          if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerbosePerformance))
-            TR_VerboseLog::writeLineLocked(TR_Vlog_INFO, "Failed to set MADV_NOHUGEPAGE for code cache: %s: %p %zu", error, middle, coldCacheSize);
+            TR_VerboseLog::writeLineLocked(TR_Vlog_INFO, "Failed to set MADV_NOHUGEPAGE for code cache: %s: %p %zu", error, _smallPageAreaStart, coldCacheSize);
          }
       else if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerbosePerformance))
          {
-         TR_VerboseLog::writeLineLocked(TR_Vlog_INFO, "Forcing code cache cold region %p-%p to use default size memory pages",                                          middle, middle + coldCacheSize);
+         TR_VerboseLog::writeLineLocked(TR_Vlog_INFO, "Forcing code cache cold region %p-%p of size %zu to use default size memory pages",
+                                        _smallPageAreaStart, _smallPageAreaStart + coldCacheSize, coldCacheSize);
          }
 
       // If the memory segment is backed by a file, disable read-ahead
       // so that touching one byte brings a single page in
       if (codeCacheSegment->j9segment()->vmemIdentifier.allocator == OMRPORT_VMEM_RESERVE_USED_MMAP_SHM)
          {
-         if (madvise(middle, coldCacheSize, MADV_RANDOM) != 0)
+         if (madvise(_smallPageAreaStart, coldCacheSize, MADV_RANDOM) != 0)
             {
             if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerbosePerformance))
                 TR_VerboseLog::writeLineLocked(TR_Vlog_INFO, "Failed to set MADV_RANDOM for cold code cache");
             }
          }
       }
+   else // Disclaim is not allowed
+      {
+      // Note: if _smallPageAreaStart and _smallPageAreaEnd are the same, there will be no disclaim attempted
+      _smallPageAreaStart = _coldCodeAllocBase;
+      _smallPageAreaEnd = _coldCodeAllocBase;
+      }
 #endif // ifdef LINUX
 
    _manager->reportCodeLoadEvents();
@@ -822,31 +860,35 @@ J9::CodeCache::disclaim(TR::CodeCacheManager *manager, bool canDisclaimOnSwap)
    int32_t disclaimDone = 0;
 
 #ifdef LINUX
-   J9JavaVM * javaVM = jitConfig->javaVM;
-   PORT_ACCESS_FROM_JAVAVM(javaVM); // for j9vmem_supported_page_sizes
+   if ((uintptr_t)_smallPageAreaStart >= (uintptr_t)_smallPageAreaEnd)
+      return disclaimDone;
+   uint8_t *disclaimStart = _smallPageAreaStart;
+   // Some of the warm code could have been written into the small page area.
+   // We don't want to disclaim warm code.
+   if ((uintptr_t)_warmCodeAlloc > (uintptr_t)_smallPageAreaStart)
+      {
+      J9JavaVM * javaVM = jitConfig->javaVM;
+      PORT_ACCESS_FROM_JAVAVM(javaVM); // for j9vmem_supported_page_sizes
+      uintptr_t round = (uintptr_t)(j9vmem_supported_page_sizes()[0] - 1);
+      disclaimStart = (uint8_t *)(((uintptr_t)_warmCodeAlloc + round) & ~round);
+      if ((uintptr_t)disclaimStart >= (uintptr_t)_smallPageAreaEnd) // Nothing to disclaim
+         return disclaimDone;
+      }
+   TR_ASSERT_FATAL((uintptr_t)_smallPageAreaEnd >= (uintptr_t)disclaimStart, "disclaimStart is past the cold area end");
 
+   size_t disclaimSize = _smallPageAreaEnd - disclaimStart;
    bool trace = TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerbosePerformance);
-   uint8_t *disclaim_start = _coldCodeAlloc;
-   size_t pageSize = j9vmem_supported_page_sizes()[0];
-   size_t round = pageSize - 1;
-   disclaim_start = (uint8_t *)(((size_t)(disclaim_start + round)) & ~round);
-
-   if (_coldCodeAllocBase <= disclaim_start)
-      return 0;
-
-   size_t disclaim_size = (_coldCodeAllocBase - disclaim_start + round) & ~round;
-
    if (trace)
       {
-      size_t warm_size = _warmCodeAlloc - _segment->segmentBase() + sizeof(this);
+      size_t warm_size = _warmCodeAlloc - _warmCodeAllocBase;
       size_t cold_size = _coldCodeAllocBase - _coldCodeAlloc;
 
-      TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "Will disclaim cold code cache %p : coldStart=%p coldBase=%p warm_size=%zuB cold_size=%zuB cold_size/(cold_size + warm_size)=%5.2f%%",
-                               this, _coldCodeAlloc, _coldCodeAllocBase,
-                               warm_size, cold_size, cold_size * 100.0/(cold_size + warm_size));
+      TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "Disclaim code cache %p between Start=%p End=%p. coldStart=%p coldBase=%p warm_size=%zuB cold_size=%zuB cold_size/(cold_size + warm_size)=%5.2f%%",
+                                     this, disclaimStart, _smallPageAreaEnd, _coldCodeAlloc, _coldCodeAllocBase,
+                                     warm_size, cold_size, cold_size * 100.0/(cold_size + warm_size));
       }
 
-   int32_t ret = madvise((void *)disclaim_start, disclaim_size, MADV_PAGEOUT);
+   int32_t ret = madvise((void *)disclaimStart, disclaimSize, MADV_PAGEOUT);
 
    if (ret != 0)
       {

runtime/compiler/runtime/J9CodeCache.hpp

@@ -119,8 +119,12 @@ class OMR_EXTENSIBLE CodeCache : public OMR::CodeCacheConnector
    */
    void resetAllocationPointers();
 
-   uint8_t * _warmCodeAllocBase; // used to reset the allocation pointers to initial values
-   uint8_t * _coldCodeAllocBase;
+   uint8_t *_warmCodeAllocBase; // used to reset the allocation pointers to initial values
+   uint8_t *_coldCodeAllocBase;
+#ifdef LINUX
+   uint8_t *_smallPageAreaStart; // used for code cache disclaiming to remember where the small page area starts/ends
+   uint8_t *_smallPageAreaEnd;
+#endif
    };