Bug 313740 - Unable to select any window when JFace warning dialog and file browse dialog are both open (back port to 3.6.x)

7 files changed, 63 insertions, 2 deletions

diff --git a/bundles/org.eclipse.swt/Eclipse SWT PI/carbon/library/os.c b/bundles/org.eclipse.swt/Eclipse SWT PI/carbon/library/os.c
index 26a8230fad..08d6923798 100644
--- a/bundles/org.eclipse.swt/Eclipse SWT PI/carbon/library/os.c
+++ b/bundles/org.eclipse.swt/Eclipse SWT PI/carbon/library/os.c
@@ -10053,6 +10053,18 @@ JNIEXPORT jint JNICALL OS_NATIVE(NavDialogGetUserAction)
 }
 #endif
 
+#ifndef NO_NavDialogGetWindow
+JNIEXPORT jint JNICALL OS_NATIVE(NavDialogGetWindow)
+	(JNIEnv *env, jclass that, jint arg0)
+{
+	jint rc = 0;
+	OS_NATIVE_ENTER(env, that, NavDialogGetWindow_FUNC);
+	rc = (jint)NavDialogGetWindow((NavDialogRef)arg0);
+	OS_NATIVE_EXIT(env, that, NavDialogGetWindow_FUNC);
+	return rc;
+}
+#endif
+
 #ifndef NO_NavDialogRun
 JNIEXPORT jint JNICALL OS_NATIVE(NavDialogRun)
 	(JNIEnv *env, jclass that, jint arg0)
diff --git a/bundles/org.eclipse.swt/Eclipse SWT PI/carbon/library/os_stats.c b/bundles/org.eclipse.swt/Eclipse SWT PI/carbon/library/os_stats.c
index 00e8dbd63f..3bc62d6d91 100644
--- a/bundles/org.eclipse.swt/Eclipse SWT PI/carbon/library/os_stats.c
+++ b/bundles/org.eclipse.swt/Eclipse SWT PI/carbon/library/os_stats.c
@@ -14,8 +14,8 @@
 
 #ifdef NATIVE_STATS
 
-int OS_nativeFunctionCount = 1012;
-int OS_nativeFunctionCallCount[1012];
+int OS_nativeFunctionCount = 1013;
+int OS_nativeFunctionCallCount[1013];
 char * OS_nativeFunctionNames[] = {
 	"AECoerceDesc",
 	"AECountItems",
@@ -714,6 +714,7 @@ char * OS_nativeFunctionNames[] = {
 	"NavDialogGetReply",
 	"NavDialogGetSaveFileName",
 	"NavDialogGetUserAction",
+	"NavDialogGetWindow",
 	"NavDialogRun",
 	"NavDialogSetFilterTypeIdentifiers",
 	"NavDialogSetSaveFileName",
diff --git a/bundles/org.eclipse.swt/Eclipse SWT PI/carbon/library/os_stats.h b/bundles/org.eclipse.swt/Eclipse SWT PI/carbon/library/os_stats.h
index 9f84601740..8bc2379672 100644
--- a/bundles/org.eclipse.swt/Eclipse SWT PI/carbon/library/os_stats.h
+++ b/bundles/org.eclipse.swt/Eclipse SWT PI/carbon/library/os_stats.h
@@ -722,6 +722,7 @@ typedef enum {
 	NavDialogGetReply_FUNC,
 	NavDialogGetSaveFileName_FUNC,
 	NavDialogGetUserAction_FUNC,
+	NavDialogGetWindow_FUNC,
 	NavDialogRun_FUNC,
 	NavDialogSetFilterTypeIdentifiers_FUNC,
 	NavDialogSetSaveFileName_FUNC,
diff --git a/bundles/org.eclipse.swt/Eclipse SWT PI/carbon/org/eclipse/swt/internal/carbon/OS.java b/bundles/org.eclipse.swt/Eclipse SWT PI/carbon/org/eclipse/swt/internal/carbon/OS.java
index 491a14e5a9..84c7722390 100644
--- a/bundles/org.eclipse.swt/Eclipse SWT PI/carbon/org/eclipse/swt/internal/carbon/OS.java
+++ b/bundles/org.eclipse.swt/Eclipse SWT PI/carbon/org/eclipse/swt/internal/carbon/OS.java
@@ -4088,6 +4088,8 @@ public static final native void NavDialogDispose(int dialogHandle);
 public static final native int NavDialogGetSaveFileName(int dialogHandle);
 /** @param dialogHandle cast=(NavDialogRef) */
 public static final native int NavDialogGetUserAction(int dialogHandle);
+/** @param dialog cast=(NavDialogRef) */
+public static final native int NavDialogGetWindow(int dialog);
 /** @param dialogHandle cast=(NavDialogRef) */
 public static final native int NavDialogRun(int dialogHandle);
 /**
diff --git a/bundles/org.eclipse.swt/Eclipse SWT/carbon/org/eclipse/swt/widgets/DirectoryDialog.java b/bundles/org.eclipse.swt/Eclipse SWT/carbon/org/eclipse/swt/widgets/DirectoryDialog.java
index 4a4b280708..6ffb960ee7 100644
--- a/bundles/org.eclipse.swt/Eclipse SWT/carbon/org/eclipse/swt/widgets/DirectoryDialog.java
+++ b/bundles/org.eclipse.swt/Eclipse SWT/carbon/org/eclipse/swt/widgets/DirectoryDialog.java
@@ -35,6 +35,7 @@ import org.eclipse.swt.*;
  * @noextend This class is not intended to be subclassed by clients.
  */
 public class DirectoryDialog extends Dialog {
+	int dialog;
 	String message = "", filterPath = "";
 
 /**
@@ -163,9 +164,11 @@ public String open () {
 			}
 		}
 		Display display = parent != null ? parent.getDisplay() : Display.getCurrent();
+		dialog = outDialog[0];
 		display.setModalDialog(this);
 		OS.NavDialogRun (outDialog [0]);
 		display.setModalDialog(null);
+		dialog = 0;
 		if (OS.NavDialogGetUserAction (outDialog [0]) == OS.kNavUserActionChoose) {
 			NavReplyRecord record = new NavReplyRecord ();
 			OS.NavDialogGetReply (outDialog [0], record);
diff --git a/bundles/org.eclipse.swt/Eclipse SWT/carbon/org/eclipse/swt/widgets/FileDialog.java b/bundles/org.eclipse.swt/Eclipse SWT/carbon/org/eclipse/swt/widgets/FileDialog.java
index 9dfd562f89..b3d4ee6461 100644
--- a/bundles/org.eclipse.swt/Eclipse SWT/carbon/org/eclipse/swt/widgets/FileDialog.java
+++ b/bundles/org.eclipse.swt/Eclipse SWT/carbon/org/eclipse/swt/widgets/FileDialog.java
@@ -37,6 +37,7 @@ import org.eclipse.swt.internal.carbon.*;
  * @noextend This class is not intended to be subclassed by clients.
  */
 public class FileDialog extends Dialog {
+	int dialog;
 	String [] filterNames = new String [0];
 	String [] filterExtensions = new String [0];
 	String [] fileNames = new String[0];	
@@ -362,9 +363,11 @@ public String open () {
 			OS.NavCustomControl (outDialog [0], OS.kNavCtlSelectCustomType, spec);
 		}
 		Display display = parent != null ? parent.getDisplay() : Display.getCurrent();
+		dialog = outDialog[0];
 		display.setModalDialog(this);
 		OS.NavDialogRun (outDialog [0]);
 		display.setModalDialog(null);
+		dialog = 0;
 		int action = OS.NavDialogGetUserAction (outDialog [0]);
 		switch (action) {
 			case OS.kNavUserActionOpen:
diff --git a/bundles/org.eclipse.swt/Eclipse SWT/carbon/org/eclipse/swt/widgets/Shell.java b/bundles/org.eclipse.swt/Eclipse SWT/carbon/org/eclipse/swt/widgets/Shell.java
index 8d3b89ede7..14f4e98399 100644
--- a/bundles/org.eclipse.swt/Eclipse SWT/carbon/org/eclipse/swt/widgets/Shell.java
+++ b/bundles/org.eclipse.swt/Eclipse SWT/carbon/org/eclipse/swt/widgets/Shell.java
@@ -641,6 +641,7 @@ void deregister () {
 }
 
 void destroyWidget () {
+	setWindowModal (display.getModalDialog (), false, true);
 	int theWindow = shellHandle;
 	/*
 	* Bug in the Macintosh.  Under certain circumstances, yet to
@@ -1914,6 +1915,42 @@ public void setVisible (boolean visible) {
 	setWindowVisible (visible);
 }
 
+void setWindowModal (Dialog dialog, boolean modal, boolean destroy) {
+	if (dialog == null) return;
+	if ((style & (SWT.APPLICATION_MODAL | SWT.SYSTEM_MODAL | SWT.PRIMARY_MODAL)) == 0) return;
+	if (isDisposed()) return;
+	/*
+	* Bug in Carbon.  For some reason, when a modal shell is opened while a file dialog is showing, neither
+	* window can be dismissed.  The fix is to temporarily change the modality of the file dialog and put the
+	* modal shell in the same group of the file dialog. 
+	*/
+	int dialogHandle = 0;
+	if (dialog instanceof FileDialog) dialogHandle = ((FileDialog)dialog).dialog;
+	if (dialog instanceof DirectoryDialog) dialogHandle = ((DirectoryDialog)dialog).dialog;
+	if (dialogHandle == 0) return;
+	if (modal) {
+		OS.SetWindowModality (OS.NavDialogGetWindow (dialogHandle), OS.kWindowModalityNone, 0);
+		int windowClass = (style & SWT.TITLE) != 0 ? OS.kMovableModalWindowClass : OS.kModalWindowClass;
+		OS.SetWindowGroup (shellHandle, OS.GetWindowGroupOfClass (windowClass));
+		OS.SelectWindow (shellHandle);
+	} else {
+		if (!destroy) {
+			int parentGroup;
+			if ((style & SWT.ON_TOP) != 0) {
+				parentGroup = OS.GetWindowGroupOfClass (OS.kFloatingWindowClass);		
+			} else {
+				if (parent != null) {
+					parentGroup = parent.getShell ().windowGroup;
+				} else {
+					parentGroup = OS.GetWindowGroupOfClass (OS.kDocumentWindowClass);
+				}
+			}
+			OS.SetWindowGroup (shellHandle, parentGroup);
+		}
+		OS.SetWindowModality (OS.NavDialogGetWindow (dialogHandle), OS.kWindowModalityAppModal, 0);
+	}
+}
+
 void setWindowVisible (boolean visible) {
 	if (OS.IsWindowVisible (shellHandle) == visible) return;
 	if (visible) {
@@ -1933,6 +1970,7 @@ void setWindowVisible (boolean visible) {
 			OS.SetWindowModality (shellHandle, inModalKind, inUnavailableWindow);
 			if (inUnavailableWindow != 0) OS.CollapseWindow (inUnavailableWindow, false);
 		}
+		setWindowModal(display.getModalDialog(), true, false);
 		int topHandle = topHandle ();
 		OS.SetControlVisibility (topHandle, true, false);
 		int [] scope = new int [1];
@@ -1985,6 +2023,7 @@ void setWindowVisible (boolean visible) {
 			}
 		}
 	} else {
+		setWindowModal(display.getModalDialog(), false, false);
 		/*
 		* Bug in the Macintosh.  Under certain circumstances, yet to
 		* be determined, calling HideWindow() and then DisposeWindow()