8196998: Create devkit for Linux with gcc 7.3

Reviewed-by: tbell, ihse

make/devkit/Makefile

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -23,7 +23,7 @@
 # questions.
 #
 
-##########################################################################################
+################################################################################
 #
 # This Makefile, together with Tools.gmk, can be used to compile a set of
 # gcc based cross compilation, portable, self contained packages, capable
@@ -31,60 +31,55 @@
 #
 # In addition to the makefiles, access to Oracle Linux installation
 # media is required. This has been tested against Oracle Enterprise Linux
-# 5.5. Set variables RPM_DIR_x86_64 and RPM_DIR_i686 respectively to point
+# 6.4.
-# to directory containing the RPMs.
 #
-# By default this Makefile will build crosstools for:
+# By default this Makefile will build a native toolchain for the current
-# * i686-unknown-linux-gnu
+# platform if called with something like this:
-# * x86_64-unknown-linux-gnu
-# The x86_64 version of the compilers will work in multi arch mode and will
-# be able to compile 32bit binaries with the -m32 flag. This makes the
-# explicit cross compiler for i686 somewhat redundant and is a known issue.
 #
-# To build the full set of crosstools, use a command line looking like this:
+# make tars
 #
-# make tars RPM_DIR_x86_64=/tmp/oel64-x86_64/Packages/ RPM_DIR_i686=/tmp/oel64-i686/Packages/
+# To build the full set of crosstools for additional platforms, use a command
+# line looking like this:
 #
-# To create a x86_64 package without the redundant i686 cross compiler, do
+# make tars platforms="x86_64-unknown-linux-gnu sparc64-unknown-linux-gnu"
-# like this:
 #
-# make tars platforms=x86_64-unknown-linux-gnu RPM_DIR_x86_64=/tmp/oel64-x86_64/Packages/ RPM_DIR_i686=/tmp/oel64-i686/Packages/
+# This is the makefile which iterates over all host and target platforms.
-
-#
-# Main makefile which iterates over all host and target platforms.
 #
 
 os := $(shell uname -o)
-cpu := x86_64
+cpu := $(shell uname -p)
-#$(shell uname -p)
-
-#
-# This wrapper script can handle exactly these platforms
-#
-platforms := $(foreach p,x86_64 i686,$(p)-unknown-linux-gnu)
-#platforms := $(foreach p,x86_64,$(p)-unknown-linux-gnu)
 
 # Figure out what platform this is building on.
 me := $(cpu)-$(if $(findstring Linux,$(os)),unknown-linux-gnu)
 
 $(info Building on platform $(me))
 
+#
+# By default just build for the current platform, which is assumed to be Linux
+#
+platforms := $(me)
+host_platforms := $(platforms)
+target_platforms := $(platforms)
+$(info host_platforms $(host_platforms))
+$(info target_platforms $(target_platforms))
+
 all compile : $(platforms)
 
 ifeq (,$(SKIP_ME))
   $(foreach p,$(filter-out $(me),$(platforms)),$(eval $(p) : $$(me)))
 endif
 
-OUTPUTDIR = $(abspath ../../build/devkit)
+OUTPUT_ROOT = $(abspath ../../build/devkit)
-RESULT = $(OUTPUTDIR)/result
+RESULT = $(OUTPUT_ROOT)/result
 
 submakevars = HOST=$@ BUILD=$(me) \
     RESULT=$(RESULT) PREFIX=$(RESULT)/$@ \
-    OUTPUTDIR=$(OUTPUTDIR)
+    OUTPUT_ROOT=$(OUTPUT_ROOT)
-$(platforms) :
+$(host_platforms) :
 	@echo 'Building compilers for $@'
-	@echo 'Targets: $(platforms)'
+	@echo 'Targets: $(target_platforms)'
-	for p in $@ $(filter-out $@,$(platforms)); do \
+	for p in $(filter $@, $(target_platforms)) $(filter-out $@, $(target_platforms)); do \
+	  $(MAKE) -f Tools.gmk download-rpms $(submakevars) TARGET=$$p && \
 	  $(MAKE) -f Tools.gmk all $(submakevars) \
 	      TARGET=$$p || exit 1 ; \
 	done
@@ -92,12 +87,8 @@ $(platforms) :
 	$(MAKE) -f Tools.gmk ccache $(submakevars) TARGET=$@
 	@echo 'All done"'
 
-$(foreach a,i686 x86_64,$(eval $(a) : $(filter $(a)%,$(platforms))))
-
-ia32 : i686
 today := $(shell date +%Y%m%d)
 
-
 define Mktar
   $(1)_tar = $$(RESULT)/sdk-$(1)-$$(today).tar.gz
   $$($(1)_tar) : PLATFORM = $(1)
@@ -105,7 +96,7 @@ define Mktar
   $$($(1)_tar) : $(1) $$(shell find $$(RESULT)/$(1))
 endef
 
-$(foreach p,$(platforms),$(eval $(call Mktar,$(p))))
+$(foreach p,$(host_platforms),$(eval $(call Mktar,$(p))))
 
 tars : all $(TARFILES)
 onlytars : $(TARFILES)
@@ -115,7 +106,9 @@ onlytars : $(TARFILES)
 	touch $@
 
 clean :
-	rm -rf build result
+	rm -rf $(addprefix ../../build/devkit/, result $(host_platforms))
+dist-clean: clean
+	rm -rf $(addprefix ../../build/devkit/, src download)
 
 FORCE :
-.PHONY : $(configs) $(platforms)
+.PHONY : all compile tars $(configs) $(host_platforms) clean dist-clean

make/devkit/Tools.gmk

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2013, 2016, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -44,24 +44,42 @@ $(info HOST=$(HOST))
 $(info BUILD=$(BUILD))
 
 ARCH := $(word 1,$(subst -, ,$(TARGET)))
+$(info ARCH=$(ARCH))
 
 ##########################################################################################
 # Define external dependencies
 
 # Latest that could be made to work.
-gcc_ver := gcc-4.9.2
+GCC_VER := 7.3.0
-binutils_ver := binutils-2.25
+ifeq ($(GCC_VER), 7.3.0)
-ccache_ver := ccache-3.2.1
+  gcc_ver := gcc-7.3.0
-mpfr_ver := mpfr-3.0.1
+  binutils_ver := binutils-2.30
-gmp_ver := gmp-4.3.2
+  ccache_ver := ccache-3.3.6
-mpc_ver := mpc-1.0.1
+  mpfr_ver := mpfr-3.1.5
+  gmp_ver := gmp-6.1.2
+  mpc_ver := mpc-1.0.3
+  gdb_ver := gdb-8.1
+else ifeq ($(GCC_VER), 4.9.2)
+  gcc_ver := gcc-4.9.2
+  binutils_ver := binutils-2.25
+  ccache_ver := ccache-3.2.1
+  mpfr_ver := mpfr-3.0.1
+  gmp_ver := gmp-4.3.2
+  mpc_ver := mpc-1.0.1
+  gdb_ver := gdb-7.12.1
+else
+  $(error Unsupported GCC version)
+endif
 
-GCC := http://ftp.gnu.org/pub/gnu/gcc/$(gcc_ver)/$(gcc_ver).tar.bz2
+GCC := http://ftp.gnu.org/pub/gnu/gcc/$(gcc_ver)/$(gcc_ver).tar.xz
-BINUTILS := http://ftp.gnu.org/pub/gnu/binutils/$(binutils_ver).tar.bz2
+BINUTILS := http://ftp.gnu.org/pub/gnu/binutils/$(binutils_ver).tar.xz
-CCACHE := http://samba.org/ftp/ccache/$(ccache_ver).tar.gz
+CCACHE := https://samba.org/ftp/ccache/$(ccache_ver).tar.xz
 MPFR := http://www.mpfr.org/${mpfr_ver}/${mpfr_ver}.tar.bz2
 GMP := http://ftp.gnu.org/pub/gnu/gmp/${gmp_ver}.tar.bz2
-MPC := http://www.multiprecision.org/mpc/download/${mpc_ver}.tar.gz
+MPC := http://ftp.gnu.org/pub/gnu/mpc/${mpc_ver}.tar.gz
+GDB := http://ftp.gnu.org/gnu/gdb/${gdb_ver}.tar.xz
+
+OEL_URL := http://yum.oracle.com/repo/OracleLinux/OL6/4/base/$(ARCH)/
 
 # RPMs in OEL6.4
 LINUX_VERSION := OEL6.4
@@ -85,30 +103,9 @@ RPM_LIST := \
     libgcc \
     zlib zlib-devel \
     libffi libffi-devel \
-    fontconfig fontconfig-devel
+    fontconfig fontconfig-devel \
-
+    systemtap-sdt-devel \
-ifeq ($(ARCH),x86_64)
+    #
-  RPM_DIR ?= $(RPM_DIR_x86_64)
-  RPM_ARCHS := x86_64 noarch
-  ifeq ($(BUILD),$(HOST))
-    ifeq ($(TARGET),$(HOST))
-      # When building the native compiler for x86_64, enable mixed mode.
-      RPM_ARCHS += i386 i686
-    endif
-  endif
-else
-  RPM_DIR ?= $(RPM_DIR_i686)
-  RPM_ARCHS := i386 i686
-endif
-
-# Sort to remove duplicates
-RPM_FILE_LIST := $(sort $(foreach a,$(RPM_ARCHS),$(wildcard $(patsubst %,$(RPM_DIR)/%*$a.rpm,$(RPM_LIST)))))
-
-#$(info RPM_FILE_LIST $(RPM_FILE_LIST))
-
-ifeq ($(RPM_FILE_LIST),)
-  $(error Found no RPMs, RPM_DIR must point to list of directories to search for RPMs)
-endif
 
 ##########################################################################################
 # Define common directories and files
@@ -122,13 +119,14 @@ else
 endif
 
 # Define directories
-RESULT := $(OUTPUTDIR)/result
+RESULT := $(OUTPUT_ROOT)/result
-BUILDDIR := $(OUTPUTDIR)/$(HOST)/$(TARGET)
+BUILDDIR := $(OUTPUT_ROOT)/$(HOST)/$(TARGET)
 PREFIX := $(RESULT)/$(HOST)
 TARGETDIR := $(PREFIX)/$(TARGET)
 SYSROOT := $(TARGETDIR)/sysroot
-DOWNLOAD := $(OUTPUTDIR)/download
+DOWNLOAD := $(OUTPUT_ROOT)/download
-SRCDIR := $(OUTPUTDIR)/src
+DOWNLOAD_RPMS := $(DOWNLOAD)/rpms
+SRCDIR := $(OUTPUT_ROOT)/src
 
 # Marker file for unpacking rpms
 rpms := $(SYSROOT)/rpms_unpacked
@@ -136,6 +134,16 @@ rpms := $(SYSROOT)/rpms_unpacked
 # Need to patch libs that are linker scripts to use non-absolute paths
 libs := $(SYSROOT)/libs_patched
 
+################################################################################
+# Download RPMs
+download-rpms:
+	mkdir -p $(DOWNLOAD_RPMS)
+        # Only run this if rpm dir is empty.
+        ifeq ($(wildcard $(DOWNLOAD_RPMS)/*.rpm), )
+	  cd $(DOWNLOAD_RPMS) && \
+	      wget -r -np -nd $(patsubst %, -A "*%*.rpm", $(RPM_LIST)) $(OEL_URL)
+        endif
+
 ##########################################################################################
 # Unpack source packages
 
@@ -147,7 +155,7 @@ define Download
 
   $$($(1)_CFG) : $$($(1)_FILE)
 	mkdir -p $$(SRCDIR)
-	tar -C $$(SRCDIR) -x$$(if $$(findstring .gz, $$<),z,j)f $$<
+	tar -C $$(SRCDIR) -xf $$<
 	$$(foreach p,$$(abspath $$(wildcard $$(notdir $$($(1)_DIR)).patch)), \
 	  echo PATCHING $$(p) ; \
 	  patch -d $$($(1)_DIR) -p1 -i $$(p) ; \
@@ -159,15 +167,32 @@ define Download
 endef
 
 # Download and unpack all source packages
-$(foreach p,GCC BINUTILS CCACHE MPFR GMP MPC,$(eval $(call Download,$(p))))
+$(foreach p,GCC BINUTILS CCACHE MPFR GMP MPC GDB,$(eval $(call Download,$(p))))
 
 ##########################################################################################
 # Unpack RPMS
 
+ifeq ($(ARCH),x86_64)
+  RPM_ARCHS := x86_64 noarch
+  ifeq ($(BUILD),$(HOST))
+    ifeq ($(TARGET),$(HOST))
+      # When building the native compiler for x86_64, enable mixed mode.
+      RPM_ARCHS += i386 i686
+    endif
+  endif
+else ifeq ($(ARCH),i686))
+  RPM_ARCHS := i386 i686
+else
+  RPM_ARCHS := $(ARCH)
+endif
+
+RPM_FILE_LIST := $(sort $(foreach a, $(RPM_ARCHS), \
+  $(wildcard $(patsubst %,$(DOWNLOAD_RPMS)/%*$a.rpm,$(RPM_LIST))) \
+))
+
 # Note. For building linux you should install rpm2cpio.
 define unrpm
-  $(SYSROOT)/$(notdir $(1)).unpacked \
+  $(SYSROOT)/$(notdir $(1)).unpacked : $(1)
-    : $(1)
   $$(rpms) : $(SYSROOT)/$(notdir $(1)).unpacked
 endef
 
@@ -210,19 +235,20 @@ $(libs) : $(rpms)
 ##########################################################################################
 # Create links for ffi header files so that they become visible by default when using the
 # devkit.
+ifeq ($(ARCH), x86_64)
+  $(SYSROOT)/usr/include/ffi.h: $(rpms)
+	cd $(@D) && rm -f $(@F) && ln -s ../lib/libffi-*/include/$(@F) .
 
-$(SYSROOT)/usr/include/ffi.h: $(rpms)
+  $(SYSROOT)/usr/include/ffitarget.h: $(rpms)
-	cd $(@D) && rm $(@F) && ln -s ../lib/libffi-*/include/$(@F) .
+	cd $(@D) && rm -f $(@F) && ln -s ../lib/libffi-*/include/$(@F) .
 
-$(SYSROOT)/usr/include/ffitarget.h: $(rpms)
+  SYSROOT_LINKS += $(SYSROOT)/usr/include/ffi.h $(SYSROOT)/usr/include/ffitarget.h
-	cd $(@D) && rm $(@F) && ln -s ../lib/libffi-*/include/$(@F) .
+endif
-
-SYSROOT_LINKS += $(SYSROOT)/usr/include/ffi.h $(SYSROOT)/usr/include/ffitarget.h
 
 ##########################################################################################
 
 # Define marker files for each source package to be compiled
-$(foreach t,binutils mpfr gmp mpc gcc ccache,$(eval $(t) = $(TARGETDIR)/$($(t)_ver).done))
+$(foreach t,binutils mpfr gmp mpc gcc ccache gdb,$(eval $(t) = $(TARGETDIR)/$($(t)_ver).done))
 
 ##########################################################################################
 
@@ -234,7 +260,8 @@ CONFIG = --target=$(TARGET) \
 PATHEXT = $(RESULT)/$(BUILD)/bin:
 
 PATHPRE = PATH=$(PATHEXT)$(PATH)
-BUILDPAR = -j16
+NUM_CORES := $(shell cat /proc/cpuinfo | grep -c processor)
+BUILDPAR = -j$(NUM_CORES)
 
 # Default commands to when making
 MAKECMD =
@@ -317,6 +344,8 @@ $(BUILDDIR)/$(binutils_ver)/Makefile \
 	      --disable-nls \
 	      --program-prefix=$(TARGET)- \
 	      --enable-multilib \
+	      --enable-gold \
+	      --enable-plugins \
 	) > $(@D)/log.config 2>&1
 	@echo 'done'
 
@@ -392,7 +421,6 @@ $(BUILDDIR)/$(gcc_ver)/Makefile \
 	      --with-sysroot=$(SYSROOT) \
 	      --enable-languages=c,c++ \
 	      --enable-shared \
-	      --enable-multilib \
 	      --disable-nls \
 	      --with-gnu-as \
 	      --with-gnu-ld \
@@ -410,6 +438,23 @@ $(BUILDDIR)/$(gcc_ver)/Makefile : $(gmp) $(mpfr) $(mpc)
 $(mpfr) : $(gmp)
 $(mpc) : $(gmp) $(mpfr)
 
+################################################################################
+# Build gdb but only where host and target match
+ifeq ($(HOST), $(TARGET))
+  $(BUILDDIR)/$(gdb_ver)/Makefile: $(GDB_CFG)
+	$(info Configuring $@. Log in $(@D)/log.config)
+	mkdir -p $(@D)
+	( \
+	  cd $(@D) ; \
+	  $(PATHPRE) $(ENVS) CFLAGS="$(CFLAGS)" $(GDB_CFG) \
+	      $(CONFIG) \
+	      --with-sysroot=$(SYSROOT) \
+	) > $(@D)/log.config 2>&1
+	@echo 'done'
+
+  $(gdb): $(gcc)
+endif
+
 ##########################################################################################
 # very straightforward. just build a ccache. it is only for host.
 $(BUILDDIR)/$(ccache_ver)/Makefile \
@@ -434,7 +479,7 @@ ifeq ($(HOST),$(TARGET))
 	  if [ ! -e $(PREFIX)/bin/$(TARGET)-$$f ]; \
 	  then \
 	    cd $(PREFIX)/bin && \
-	    ln -s $$f $(TARGET)-$$f ; \
+	    ln -fs $$f $(TARGET)-$$f ; \
 	  fi \
 	done
 	@touch $@
@@ -451,7 +496,7 @@ ifeq ($(HOST),$(TARGET))
 	  if [ ! -e $(TARGETDIR)/$(l)/$$f ]; then \
 	    mkdir -p $(TARGETDIR)/$(l) && \
 	    cd $(TARGETDIR)/$(l)/ && \
-	    ln -s $(if $(findstring /,$(l)),../,)../../$(l)/$$f $$f; \
+	    ln -fs $(if $(findstring /,$(l)),../,)../../$(l)/$$f $$f; \
 	  fi \
 	done;)
 	@echo 'done'
@@ -471,6 +516,7 @@ $(TARGETDIR)/%.done : $(BUILDDIR)/%/Makefile
 	$(PATHPRE) $(ENVS) $(MAKE) $(BUILDPAR) -f $< -C $(<D) $(MAKECMD) $(MAKECMD.$(notdir $@)) > $(<D)/log.build 2>&1
 	@echo -n 'installing...'
 	$(PATHPRE) $(MAKE) $(INSTALLPAR) -f $< -C $(<D) $(INSTALLCMD) $(MAKECMD.$(notdir $@)) > $(<D)/log.install 2>&1
+	@mkdir -p $(@D)
 	@touch $@
 	@echo 'done'
 
@@ -486,17 +532,38 @@ $(PREFIX)/devkit.info: FRC
 	echo 'DEVKIT_NAME="$(gcc_ver) - $(LINUX_VERSION)"' >> $@
 	echo 'DEVKIT_TOOLCHAIN_PATH="$$DEVKIT_ROOT/bin"' >> $@
 	echo 'DEVKIT_SYSROOT="$$DEVKIT_ROOT/$$host/sysroot"' >> $@
+	echo 'DEVKIT_EXTRA_PATH="$$DEVKIT_ROOT/bin"' >> $@
+
+##########################################################################################
+# Copy these makefiles into the root of the kit
+$(PREFIX)/Makefile: ./Makefile
+	rm -rf $@
+	cp $< $@
+
+$(PREFIX)/Tools.gmk: ./Tools.gmk
+	rm -rf $@
+	cp $< $@
+
+THESE_MAKEFILES := $(PREFIX)/Makefile $(PREFIX)/Tools.gmk
 
 ##########################################################################################
 
 ifeq ($(TARGET), $(HOST))
+  # To build with dtrace support, the build needs access to the dtrace executable from the
+  # sysroot. Generally we don't want to add binaries in the sysroot to the path, but
+  # luckily this seems to execute well enough on a different host Linux distro, so symlink
+  # it into the main bin dir.
+  $(PREFIX)/bin/dtrace:
+	@echo 'Creating dtrace soft link'
+	ln -s ../$(HOST)/sysroot/usr/bin/dtrace $@
+
   $(PREFIX)/bin/%:
 	@echo 'Creating missing $* soft link'
 	ln -s $(TARGET)-$* $@
 
   missing-links := $(addprefix $(PREFIX)/bin/, \
       addr2line ar as c++ c++filt elfedit g++ gcc gprof ld nm objcopy ranlib readelf \
-      size strings strip)
+      size strings strip ld.bfd ld.gold dtrace)
 endif
 
 ##########################################################################################
@@ -507,7 +574,9 @@ rpms : $(rpms)
 libs : $(libs)
 sysroot : rpms libs
 gcc : sysroot $(gcc) $(gccpatch)
-all : binutils gcc bfdlib $(PREFIX)/devkit.info $(missing-links) $(SYSROOT_LINKS)
+gdb : $(gdb)
+all : binutils gcc bfdlib $(PREFIX)/devkit.info $(missing-links) $(SYSROOT_LINKS) \
+    $(THESE_MAKEFILES) gdb
 
 # this is only built for host. so separate.
 ccache : $(ccache)